metanorma 2.4.3 → 2.5.2

data/lib/metanorma/collection/renderer/fileprocess.rb

@@ -4,6 +4,8 @@ require "isodoc"
 require "metanorma-utils"
 require_relative "fileparse"
 require_relative "filelocation"
+require_relative "../artifact_store"
+require "json"
 
 module Metanorma
   class Collection
@@ -16,9 +18,61 @@ module Metanorma
         opts = compile_options_base(identifier)
           .merge(compile_options_update(identifier))
 
+        err_count = @compile.errors.size
         @compile.compile file, opts
         @files.set(identifier, :outputs, {})
         file_compile_formats(filename, identifier, opts)
+        file_compile_verify(identifier, @compile.errors[err_count..] || [])
+      end
+
+      # A member whose rendering raised produces no output file, but the
+      # collection build used to continue, exit 0, and link the absent
+      # document from index.html -- silent data loss (#586). Two
+      # complementary checks close that hole. (1) Compile pools rescued
+      # error messages in @compile.errors for its caller to surface; the
+      # standalone CLI does, but the collection pipeline never read the
+      # pool. The pool is shared across members and never reset, so the
+      # slice appended during this member's compile is this member's
+      # errors -- sound while members compile sequentially (the current
+      # design; @compile's own format-level parallelism is joined before
+      # compile returns). (2) Independently of whether anything was
+      # pooled, verify the outputs registered for the member exist on
+      # disk. Failures are reported here and accumulated; the collection
+      # renders every remaining member before Collection#render aborts,
+      # so one run reports all failed members.
+      def file_compile_verify(identifier, errors)
+        # Section-split parts (marked with :parentid) register format
+        # slots they never produce -- their :xml slot in particular is
+        # never written, and under filename renaming it aliases to the
+        # parent's name -- so verifying them is a false positive. #586
+        # was scoped to the non-sectionsplit path; parts are excluded
+        # like their parents (which return early above).
+        @files.get(identifier, :parentid) and return
+        missing = (@files.get(identifier, :outputs) || {})
+          .reject { |_fmt, path| File.exist?(path) }
+        errors.empty? && missing.empty? and return
+        file_compile_failure_log(identifier, errors, missing)
+        msg = file_compile_failure_msg(identifier, errors, missing)
+        ::Metanorma::Util.log("[metanorma] Error: #{msg}", :error)
+        fatal_errors << msg
+      end
+
+      def file_compile_failure_log(identifier, errors, missing)
+        missing.each_value do |path|
+          @log&.add("METANORMA_4", nil, params: [identifier, path])
+        end
+        errors.each do |e|
+          @log&.add("METANORMA_5", nil, params: [identifier, e])
+        end
+      end
+
+      def file_compile_failure_msg(identifier, errors, missing)
+        ret = []
+        missing.empty? or
+          ret << "expected output not generated: " \
+                 "#{missing.values.join(', ')}"
+        ret += errors
+        "Failed to render collection member #{identifier}: #{ret.join('; ')}"
       end
 
       def compile_options_base(identifier)
@@ -67,8 +121,11 @@ module Metanorma
           if @files.get(ident, :attachment) then copy_file_to_dest(ident)
           else
             file, fname = @files.targetfile_id(ident, read: true)
+            @preserve_unresolved && member_stored?(ident, file) and next
             warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
             collection_xml = update_xrefs(file, ident, internal_refs)
+            @preserve_unresolved and
+              store_member_artefacts(ident, file, collection_xml)
             # Strip .xml or .html extension, but NOT section numbers like .0
             fname_base = File.basename(fname)
             collection_filename = fname_base
@@ -83,6 +140,31 @@ module Metanorma
         end
       end
 
+      # Persist an isolated member's stub-bearing Semantic XML and its anchor
+      # contribution to the content-addressed store, so a later reinflation pass
+      # can relink the stubs and a re-run can resume without recompiling.
+      # Within-campaign content key for a member. (Cross-build soundness needs
+      # the full input closure -- source + includes + templates + schemas +
+      # versions; see the plan gotcha. Here the source bytes + tool version
+      # suffice because inputs are frozen across a single build campaign.)
+      def member_content_hash(source)
+        ArtifactStore.content_hash(source, ::Metanorma::VERSION)
+      end
+
+      # Resume predicate: is this member already compiled and stored for its
+      # current input content? If so the isolated compile can skip it.
+      def member_stored?(ident, source)
+        @artifact_store.key?(ident, member_content_hash(source), :semantic)
+      end
+
+      def store_member_artefacts(ident, source, semantic_xml)
+        hash = member_content_hash(source)
+        @artifact_store.write(ident, hash, :semantic, semantic_xml)
+        @artifact_store.write(ident, hash, :anchors,
+                              JSON.generate(@files.get(ident, :anchors) || {}))
+        @keep_cache or @artifact_store.prune_superseded(ident, hash)
+      end
+
       # gather internal bibitem references
       def gather_internal_refs
         @files.keys.each_with_object({}) do |i, refs|
@@ -168,7 +250,7 @@ module Metanorma
       end
 
       def update_bibitem(bib, identifier)
-        newbib, url = update_bibitem_prep(bib, identifier)
+        newbib, url, attachment = update_bibitem_prep(bib, identifier)
         newbib or return
         dest = begin
           newbib.at("./docidentifier") || newbib.at(ns("./docidentifier"))
@@ -176,28 +258,63 @@ module Metanorma
           nil
         end
         dest or dest = newbib.elements[-1]
+        # An attachment carries both a <uri type="attachment"> (which drives
+        # isodoc's fmt-link attachment="true") and the citation uri; the
+        # attachment uri goes first. metanorma/iso-10303#208.
+        attachment and dest.previous = "<uri type='attachment'>#{url}</uri>"
         dest.previous = "<uri type='citation'>#{url}</uri>"
         bib.replace(newbib)
       end
 
       def update_bibitem_prep(bib, identifier)
-        docid = get_bibitem_docid(bib, identifier) or return [nil, nil]
+        docid = get_bibitem_docid(bib, identifier) or return [nil, nil, nil]
         newbib = dup_bibitem(docid, bib)
-        url = @files.url(docid, relative: true,
-                                doc: !@files.get(docid, :attachment))
-        # Use :outputs[:html] if available (after compilation),
-        # otherwise convert :out_path to HTML (before compilation)
-        current_html = @files.get(identifier, :outputs)&.dig(:html)
-        if !current_html && (out_path = @files.get(identifier, :out_path))
-          # Convert .xml to .html, following same logic as ref_file_xml2html
-          current_html = if out_path.end_with?(".xml")
-                           out_path.sub(/\.xml$/, ".html")
-                         else
-                           "#{out_path}.html"
-                         end
-        end
+        attachment = @files.get(docid, :attachment)
+        url = @files.url(docid, relative: true, doc: !attachment)
+        current_html = referencing_html_location(identifier, attachment)
         url = make_relative_path(current_html, url) if current_html
-        [newbib, url]
+        [newbib, url, attachment]
+      end
+
+      # Location of the HTML that will carry the cross-reference to the target,
+      # used as the base for relativising the attachment/citation URL.
+      #
+      # Normally this is the referencing document's own output (:outputs[:html]
+      # after compilation, else :out_path converted to .html before it).
+      #
+      # But when the referencing document is sectionsplit, its content is not
+      # emitted at its own out_path: it is split into section files written at
+      # the sectionsplit output location (the collection root, unless a
+      # sectionsplit_filename directory is configured). Relativising an
+      # attachment URL against the unsplit out_path then overshoots -- the
+      # ../../ climbs above the split file and the attachment link breaks
+      # (https://github.com/metanorma/iso-10303/issues/208). So for attachment
+      # targets of a sectionsplit document, base the relative path on the
+      # sectionsplit output directory instead.
+      def referencing_html_location(identifier, attachment)
+        attachment && @files.get(identifier, :sectionsplit) and
+          return sectionsplit_ref_html(identifier)
+        document_ref_html(identifier)
+      end
+
+      # The split section files sit at the sectionsplit output directory: the
+      # collection root, or the directory of a configured sectionsplit_filename.
+      def sectionsplit_ref_html(identifier)
+        d = @files.preserve_directory_structure?(identifier)
+        d ? File.join(File.dirname(d), "x.html") : "x.html"
+      end
+
+      # Use :outputs[:html] if available (after compilation), otherwise convert
+      # :out_path to HTML (before compilation), following ref_file_xml2html.
+      def document_ref_html(identifier)
+        @files.get(identifier, :outputs)&.dig(:html) ||
+          out_path_to_html(@files.get(identifier, :out_path))
+      end
+
+      def out_path_to_html(out_path)
+        out_path or return nil
+        out_path.end_with?(".xml") and return out_path.sub(/\.xml$/, ".html")
+        "#{out_path}.html"
       end
     end
   end

data/lib/metanorma/collection/renderer/renderer.rb

@@ -17,9 +17,35 @@ module Metanorma
     class Renderer
       FORMATS = %i[html xml doc pdf pdf-portfolio].freeze
 
-      attr_accessor :isodoc, :isodoc_presxml, :nested
+      attr_accessor :isodoc, :isodoc_presxml
       attr_reader :xml, :compile, :compile_options, :documents, :outdir,
-                  :manifest
+                  :manifest, :fatal_errors
+
+      # Run the block with the renderer in nested mode, restoring the previous
+      # mode afterwards. Nested mode (used by sectionsplit, which re-enters this
+      # renderer's update_xrefs on the pre-split document) preserves unresolved
+      # erefs and skips the finalising reference passes.
+      def with_nested
+        saved = @nested
+        @nested = true
+        yield
+      ensure
+        @nested = saved
+      end
+
+      # Run the block with the renderer preserving unresolved cross-document
+      # references as stubs instead of stripping them -- for an isolated build of
+      # a collection member (compiled without the rest of its collection
+      # present). Unlike +with_nested+, the ordinary intra-document passes still
+      # run (xref_process, svgmap); only the cross-document resolve/strip is
+      # turned into preserve, so a later reinflation pass can relink the stubs.
+      def with_preserve_unresolved
+        saved = @preserve_unresolved
+        @preserve_unresolved = true
+        yield
+      ensure
+        @preserve_unresolved = saved
+      end
 
       # This is only going to render the HTML collection
       # @param xml [Metanorma::Collection] input XML collection
@@ -67,10 +93,39 @@ module Metanorma
         @final = collection.final
         @c = HTMLEntities.new
         @files_to_delete = []
+        # Per-member rendering failures (missing outputs, pooled compile
+        # errors), accumulated by file_compile_verify; Collection#render
+        # aborts on them once every member has rendered (#586).
+        @fatal_errors = []
         @nested = options[:nested]
         # if false, this is the root instance of Renderer
         # if true, then this is not the last time Renderer will be run
         # (e.g. this is sectionsplit)
+        # Isolated/incremental build: preserve unresolved cross-document
+        # references as stubs instead of resolving or stripping them, so each
+        # member compiles independently and a later reinflation pass relinks.
+        @preserve_unresolved = options[:preserve_unresolved]
+        # Durable content-addressed store for staged artefacts. Opt-in: a plain
+        # build sets neither :preserve_unresolved nor :artifact_store_dir, so it
+        # gets the no-op Null store and is unaffected. Staging (:preserve_unresolved)
+        # defaults the store directory to ArtifactStore::DEFAULT_DIRNAME when no
+        # :artifact_store_dir is given, so callers need not name it. (Reinflation
+        # reads stored stubs via the manifest, not this object, so it does not
+        # trigger the default -- that would only create an empty directory.)
+        @artifact_store =
+          if (dir = options[:artifact_store_dir]) || options[:preserve_unresolved]
+            ArtifactStore.new(dir || ArtifactStore::DEFAULT_DIRNAME)
+          else
+            NullArtifactStore.new
+          end
+        # Reinflation pass: the input is a stored stub-bearing Semantic XML that
+        # already has intra-doc xrefs and the document suffix applied, so run
+        # only the cross-document resolution, not the passes already done.
+        @reinflate = options[:reinflate]
+        # Cache retention: by default keep only each document's latest artefacts,
+        # pruning superseded content-hash versions on write; keep_cache retains
+        # every version (history / multiple states side by side).
+        @keep_cache = options[:keep_cache]
 
         @coverpage = options[:coverpage] || collection.coverpage
         @coverpage_pdf_portflio = options[:coverpage_pdf_portfolio] ||
@@ -90,21 +145,6 @@ module Metanorma
         directives_normalise_keystore_pdf_portfolio(directives)
       end
 
-      # KILL
-      def directives_normalise_coverpage_pdf_portfolio(directives)
-        directives.reject! { |d| d.key == "coverpage-pdf-portfolio" }
-        absolute_coverpage_pdf_portflio = @coverpage_pdf_portflio &&
-          Pathname.new(@coverpage_pdf_portflio).absolute?
-        @coverpage_pdf_portflio =
-          Util::rel_path_resolve(@dirname, @coverpage_pdf_portflio)
-        absolute_coverpage_pdf_portflio or
-          @coverpage_pdf_portflio = Pathname.new(@coverpage_pdf_portflio)
-            .relative_path_from(Pathname.new(@outdir)).to_s
-        directives << ::Metanorma::Collection::Config::Directive
-          .new(key: "coverpage-pdf-portfolio", value: @coverpage_pdf_portflio)
-        directives
-      end
-
       def directives_normalise_coverpage_pdf_portfolio(directives)
         directives_resolve_filepath(directives, "coverpage-pdf-portfolio",
                                     @coverpage_pdf_portflio)
@@ -258,10 +298,25 @@ module Metanorma
           .new(key: "documents-inline")
         out.bibdatas.each_key do |ident|
           id = @isodoc.docid_prefix(nil, ident.dup)
-          @files.get(id, :attachment) || @files.get(id, :outputs).nil? and next
+          @files.get(id, :attachment) and next
+          # A non-attachment document with no compiled output for THIS format
+          # cannot be inlined: leaving it in place serialises as a bibdata-only,
+          # namespace-less <metanorma> that mn2pdf silently drops. Skip loudly
+          # rather than crash on a nil path or corrupt the collection PDF.
+          # (Sectionsplit parents get a :presentation output upstream, so they
+          # inline whole for the PDF/presentation path.)
+          outputs = @files.get(id, :outputs)
+          # The registered path may exist without the file: a member whose
+          # rendering failed registers its intended outputs but writes
+          # nothing (#586), so check the disk, not just the registration.
+          if outputs.nil? || outputs[ext].nil? || !File.exist?(outputs[ext])
+            warn "[metanorma] collection document '#{id}' has no compiled " \
+                 "#{ext} output to inline; skipping (its doc-container would " \
+                 "otherwise be bibdata-only)."
+            next
+          end
           out.documents[Util::key id] =
-            Metanorma::Collection::Document
-              .raw_file(@files.get(id, :outputs)[ext])
+            Metanorma::Collection::Document.raw_file(outputs[ext])
         end
         out
       end

data/lib/metanorma/collection/sectionsplit/sectionsplit.rb

@@ -7,8 +7,24 @@ require "concurrent-ruby"
 
 module Metanorma
   class Collection
+    # Sectionsplit is a collection render nested inside a collection render:
+    # a document is split into one output file per section, and those files are
+    # then rendered as their own (sub-)collection.
+    #
+    # Two output-location invariants matter (both load-bearing -- see the
+    # collection architecture review plan and metanorma/iso-10303#208):
+    #
+    # * XML section files are always written FLAT to the split directory
+    #   (basename only). HTML output may instead carry a directory taken from
+    #   sectionsplit_filename, reattached only at HTML compile time
+    #   (preserve_directory_structure?).
+    # * A sectionsplit document's content is emitted at the sectionsplit output
+    #   location (the collection root, unless sectionsplit_filename sets a
+    #   directory), NOT at the document's own :out_path. Anything relativising a
+    #   URL for such a document (e.g. attachment/citation links) must base it on
+    #   the split location, or the ../../ overshoots.
     class Sectionsplit
-      attr_accessor :filecache, :key
+      attr_accessor :filecache, :key, :whole_presentation_file
 
       def initialize(opts)
         @input_filename = opts[:input]
@@ -40,6 +56,13 @@ module Metanorma
       # Input XML is Semantic XML
       def sectionsplit
         xml = sectionsplit_prep(File.read(@input_filename), @base, @dir)
+        # Retain the whole (un-split) Presentation XML. sectionsplit is an
+        # HTML-only mechanism, but the collection PDF/presentation path must
+        # inline each document intact; without this the sectionsplit parent
+        # reaches concatenation with no presentation output and degrades to a
+        # bibdata-only, namespace-less <metanorma> stub that mn2pdf drops.
+        # https://github.com/metanorma/iso-10303/issues/208
+        @whole_presentation_file = write_whole_presentation(xml)
         @key = Metanorma::Collection::XrefProcess::xref_preprocess(xml, @isodoc)
         empty = empty_doc(xml)
         empty1 = empty_attachments(empty)
@@ -66,11 +89,11 @@ module Metanorma
 
       def sectionsplit2(xml, empty, chunks, parentnode, opt)
         @pool.post do
-          output_filename = @sectionsplit_filename
-            &.gsub(/\{document-num\}/, @parent_idx.to_s)
-            &.gsub(/\{basename_legacy\}/, @base)
-            &.gsub(/\{basename\}/, File.basename(@base, ".*"))
-            &.gsub(/\{sectionsplit-num\}/, opt[:idx].to_s)
+          output_filename = Util.substitute_filename_pattern(
+            @sectionsplit_filename,
+            document_num: @parent_idx, basename: File.basename(@base, ".*"),
+            basename_legacy: @base, sectionsplit_num: opt[:idx]
+          )
           warn "Sectionsplit: #{output_filename}"
           a = sectionfile(xml, empty, output_filename, chunks,
                           parentnode)
@@ -108,6 +131,16 @@ module Metanorma
         r
       end
 
+      # Persist the whole, svgmap-restored Presentation XML (the in-memory doc,
+      # not the on-disk tmp which still carries the internal svgmap1 marker) so
+      # the collection PDF path can inline the document whole. Written flat to
+      # the split output directory, where the per-section files also live.
+      def write_whole_presentation(xml)
+        fname = File.join(@splitdir, "#{@base}.whole.presentation.xml")
+        File.write(fname, xml.to_xml, encoding: "UTF-8")
+        fname
+      end
+
       def sectionsplit_preprocess_semxml(file, filename)
         xml = Nokogiri::XML(file, &:huge)
         type = xml.root["flavor"]
@@ -120,10 +153,8 @@ module Metanorma
 
       def sectionsplit_update_xrefs(xml)
         if c = @fileslookup&.parent
-          n = c.nested
-          c.nested = true # so unresolved erefs are not deleted
-          c.update_xrefs(xml, @ident, {})
-          c.nested = n
+          # nested mode so unresolved erefs are not deleted
+          c.with_nested { c.update_xrefs(xml, @ident, {}) }
           xml.xpath("//xmlns:svgmap").each { |x| x.name = "svgmap1" }
           # do not process svgmap until after files are split
         end

data/lib/metanorma/collection/util/util.rb

@@ -81,6 +81,22 @@ module Metanorma
           @c.decode(ident).gsub(/(\p{Zs})+/, " ")
         end
 
+        # Substitute special strings in filename patterns. Single source for
+        # both FileLookup#substitute_filename_pattern and sectionsplit.
+        # @param pattern [String] filename pattern with placeholders
+        # @param options [Hash] substitution values:
+        #   :document_num, :basename, :basename_legacy, :sectionsplit_num
+        def substitute_filename_pattern(pattern, options = {})
+          pattern or return pattern
+          subs = { "{document-num}" => options[:document_num],
+                   "{basename}" => options[:basename],
+                   "{basename_legacy}" => options[:basename_legacy],
+                   "{sectionsplit-num}" => options[:sectionsplit_num] }
+          subs.each_with_object(pattern.dup) do |(token, val), result|
+            val and result.gsub!(token, val.to_s)
+          end
+        end
+
         def taste2flavor(taste)
           tastes = Metanorma::TasteRegister.instance.aliases
           tastes[taste.to_sym] and taste = tastes[taste.to_sym]

data/lib/metanorma/compile/render.rb

@@ -135,8 +135,7 @@ module Metanorma
                         output_paths[:out], ext, isodoc_options)
       wrap_html(options, output_paths[:ext], output_paths[:out])
     rescue StandardError => e
-      strict = ext == :presentation || isodoc_options[:strict] == true
-      isodoc_error_process(e, strict, false)
+      isodoc_error_process(e, strict_error?(ext, isodoc_options), false)
     end
 
     # Process format directly from semantic XML
@@ -145,8 +144,7 @@ module Metanorma
                         ext, isodoc_options)
       true # Return as Thread equivalent
     rescue StandardError => e
-      strict = ext == :presentation || isodoc_options[:strict] == "true"
-      isodoc_error_process(e, strict, true)
+      isodoc_error_process(e, strict_error?(ext, isodoc_options), true)
       ext != :presentation
     end
 
@@ -168,12 +166,29 @@ module Metanorma
 
     private
 
+    # A presentation failure is always strict: every downstream format
+    # needs the presentation XML. The :strict option arrives as a
+    # boolean from the CLI but as a string from document attributes, so
+    # accept both (the two rescue sites previously each checked only
+    # one form).
+    def strict_error?(ext, isodoc_options)
+      ext == :presentation ||
+        [true, "true"].include?(isodoc_options[:strict])
+    end
+
+    # On the strict path the message is pooled into @errors for the
+    # caller to surface (the standalone CLI reports the pool and
+    # aborts; the collection renderer attributes it to the failed
+    # member, #586). The class+message line is also printed here in all
+    # cases, so it sits adjacent to its backtrace in the build log: a
+    # pooled message surfaces only at end of run, and a bare backtrace
+    # mid-log is undiagnosable. The exception class is retained because
+    # for errors like Encoding::UndefinedConversionError it carries
+    # half the diagnosis.
     def isodoc_error_process(err, strict, must_abort)
-      if strict || err.message.include?("Fatal:")
-        @errors << err.message
-      else
-        puts err.message
-      end
+      msg = "#{err.class}: #{err.message}"
+      strict || err.message.include?("Fatal:") and @errors << msg
+      puts msg
       puts err.backtrace.join("\n")
       must_abort and 1
     end

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "2.4.3".freeze
+  VERSION = "2.5.2".freeze
 end

data/metanorma.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.bindir        = "bin"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 3.1.0"
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0")
 
   spec.add_runtime_dependency "asciidoctor"
   spec.add_runtime_dependency "concurrent-ruby"