metanorma 2.4.3 → 2.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 157d526ec5945e1eb8faae4f4cc75466fbbf59ec367695264edd10d22d10bd37
-  data.tar.gz: 84eb4ecb451f7614da76abba559a4fe5b750425516dc78357480b68c5d50fe16
+  metadata.gz: cc6ddbece893102c78d8b27a94f175a85c45cfb911e8d7bc908440bd19a58e85
+  data.tar.gz: 226289825a52fe8da92a744a00a2239385e46f0964468be522f6b5bb7ba8066d
 SHA512:
-  metadata.gz: 04f311680d84a53dca58be88d51f85df772f5bc75ab928393a55f37253b342ba68d0931da23142ddd45f3a3ec11bb2f778ef165878f02d790f46fca64935a022
-  data.tar.gz: b73810634afb80f8c8dc5b4613421759479b7550262645efa42357b3f53d37ddbfc5f9d2520681ecbade585703baa840b489f975a0d9c28917a095bc93a8663c
+  metadata.gz: 0d9c775a338fc4b30a27aa53e217037583228a702c61977070f7e76895718b7a1d8e138b26f3053eed4ae9bdfb361a9b4545b62ad4c6e98df5f187eef8209399
+  data.tar.gz: f5c41df1386891d848d76576e783335697c80adb75eefda918a990c414ec112844d53e9f94717049873171f9978acaa28e7eed98e04ab11eedcbb11d87119576

data/.rubocop.yml

@@ -3,8 +3,20 @@
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/main/ci/rubocop.yml
 
+# Rubocop plugins enabled centrally so every metanorma-org gem picks them up
+# on cimas sync — best practice belongs at the shared-template layer, not
+# per-repo. Per ronaldtse feedback on metanorma/ci#332.
+plugins:
+  - rubocop-rspec
+  - rubocop-performance
+  - rubocop-rake
+
 # local repo-specific modifications
 # ...
 
 AllCops:
-  TargetRubyVersion: 3.4
+  # 3.3 matches the org-wide minimum being pushed via #274 (Raise minimum
+  # Ruby version to 3.3 due to EOL of 3.2 on 2026-03-31). Was 3.4 before
+  # this commit — 3.4 was above the org's stated minimum and would have
+  # applied Rubocop rules that fail-close on gems still targeting 3.3.
+  TargetRubyVersion: 3.3

data/lib/metanorma/collection/artifact_store.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "digest"
+
+module Metanorma
+  class Collection
+    # Durable, content-addressed store for staged collection-build artefacts.
+    #
+    # Enables incremental (batched) builds and resumption across runs: an
+    # artefact is named +<docid-slug>.<content-hash>.<stage>.<format>+, so a
+    # document already built with identical inputs is detected by filename alone
+    # and reused, and an interrupted run resumes by recomputing which artefacts
+    # are already present.
+    #
+    # Reuse is keyed on the caller-supplied content hash of the document's
+    # *input closure* (source + resolved includes + templates + referenced
+    # schemas + tool versions + options) -- never on file presence alone. See
+    # metanorma/iso-10303#455 for why presence-keyed reuse of compiled XML is
+    # unsound; the store deliberately requires a content hash it does not infer.
+    #
+    # Disabled by default: the collection renderer uses NullArtifactStore unless
+    # an incremental build is explicitly requested (opt-in, never default).
+    class ArtifactStore
+      DEFAULT_DIRNAME = ".metanorma-collection-cache"
+
+      # The artefact-stage vocabulary, and the one non-redundant property each
+      # stage carries: the format it is serialised as (also its file extension).
+      # THIS IS ITS HOME: every producer and consumer names its stage from this
+      # table by symbol, never a bare string literal elsewhere; an unknown stage
+      # raises (KeyError via +fetch+), so the vocabulary cannot drift outside
+      # this file. The stage's *name* is the filename token, so it is not stored
+      # here -- that would just restate the key. The stages trace the pipeline
+      # (isolated compile -> index -> reinflate); each role is documented inline:
+      STAGE_FORMATS = {
+        # Compiled Semantic XML, unresolved cross-document repo:() references
+        # PRESERVED as deterministic stubs (neither stripped nor resolved). The
+        # unit of an isolated per-document compile; a pure function of the
+        # document's input closure, hence content-addressable and reusable.
+        semantic: "xml",
+        # This document's contribution to the global anchor index: its anchors
+        # and ids (type => {label/UUID => anchor-id}). Aggregated across all
+        # documents to build the anchor -> owning-document index used at
+        # reinflation.
+        anchors: "json",
+        # Reinflated Presentation XML: the preserved stubs resolved against the
+        # global index into real cross-document links. Shared source for the PDF
+        # and Word collection outputs, and per-document for HTML.
+        presentation: "xml",
+      }.freeze
+
+      attr_reader :dir
+
+      def initialize(dir)
+        @dir = dir
+        FileUtils.mkdir_p(@dir)
+      end
+
+      # <docid-slug>.<content-hash>.<stage>.<format>
+      # +stage+ must be a key of STAGE_FORMATS; an unknown stage raises KeyError.
+      def path(docid, content_hash, stage)
+        format = STAGE_FORMATS.fetch(stage)
+        File.join(@dir, "#{slug(docid)}.#{content_hash}.#{stage}.#{format}")
+      end
+
+      # Is this exact (document, input-closure, stage) artefact already present?
+      # This is the resumption / skip predicate: a hit is byte-identical to a
+      # fresh build because the hash covers every input that determines output.
+      def key?(docid, content_hash, stage)
+        File.exist?(path(docid, content_hash, stage))
+      end
+
+      def read(docid, content_hash, stage)
+        f = path(docid, content_hash, stage)
+        File.exist?(f) ? File.read(f, encoding: "UTF-8") : nil
+      end
+
+      # Idempotent write: the same (docid, content_hash, stage) always maps to
+      # the same path and the content is a pure function of the hash, so a re-run
+      # overwrites with identical bytes -- no side effects on repeat.
+      def write(docid, content_hash, stage, content)
+        f = path(docid, content_hash, stage)
+        File.write(f, content, encoding: "UTF-8")
+        f
+      end
+
+      # Remove every stored version of this document whose content hash is not
+      # +keep_hash+ -- the superseded artefacts left behind when the document's
+      # content changed. Keeps exactly one (current) version per document per
+      # stage, bounding disk to current state without losing the cache the next
+      # build resumes from.
+      def prune_superseded(docid, keep_hash)
+        STAGE_FORMATS.each do |stage, ext|
+          keep = path(docid, keep_hash, stage)
+          Dir[File.join(@dir, "#{slug(docid)}.*.#{stage}.#{ext}")].each do |f|
+            f == keep or File.delete(f)
+          end
+        end
+      end
+
+      # Wipe the entire store. The escape hatch when inputs the content hash
+      # cannot see -- imported EXPRESS schemas, templates, included files -- have
+      # changed, so the cache can no longer be trusted; the next build recompiles
+      # everything. Tracking such shared-input changes is the collection
+      # maintainer's responsibility, not the build's.
+      def clear
+        FileUtils.rm_rf(@dir)
+        FileUtils.mkdir_p(@dir)
+      end
+
+      # SHA256 over the input closure, truncated to 16 hex (64 bits -- ample for
+      # a per-collection store, and short enough to stay well inside filename
+      # length limits). Each input is hashed to a fixed-width digest and the
+      # digests concatenated before the final hash, so no two distinct closures
+      # collide on the pre-truncation digest. The caller assembles the closure;
+      # the store does not infer it.
+      def self.content_hash(*inputs)
+        digests = inputs.map { |i| Digest::SHA256.hexdigest(i.to_s) }
+        Digest::SHA256.hexdigest(digests.join)[0, 16]
+      end
+
+      private
+
+      def slug(docid)
+        docid.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
+          .gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
+      end
+    end
+
+    # Null object: the store disabled (the default). Never reports a hit, never
+    # writes. Lets the renderer treat "no incremental build" as the no-store
+    # path without a conditional at every call site (open/closed).
+    class NullArtifactStore
+      def path(*_args) = nil
+      def key?(*_args) = false
+      def read(*_args) = nil
+      def write(*_args) = nil
+      def prune_superseded(*_args) = nil
+      def clear(*_args) = nil
+    end
+  end
+end

data/lib/metanorma/collection/collection.rb

@@ -11,6 +11,9 @@ module Metanorma
 
   class AdocFileNotFoundException < StandardError; end
 
+  # One or more collection members failed to render (#586)
+  class RenderFailureException < StandardError; end
+
   # Metanorma collection of documents
   class Collection
     attr_reader :file, :prefatory, :final
@@ -132,8 +135,24 @@ module Metanorma
       opts[:log] = @log
       opts[:flavor] = @flavor
       output_folder(opts)
-      ::Metanorma::Collection::Renderer.render self, opts
+      cr = ::Metanorma::Collection::Renderer.render self, opts
       clean_exit
+      render_failures_abort(cr)
+      cr
+    end
+
+    # A member whose rendering fails no longer lets the collection
+    # report success with the document missing from the output (#586).
+    # Every member is rendered before aborting, so one run reports all
+    # failures; the raise makes callers (CLI, suma) exit non-zero. Runs
+    # after clean_exit so the failures are in the written log.
+    def render_failures_abort(renderer)
+      f = renderer.fatal_errors
+      f.empty? and return
+      raise RenderFailureException.new(
+        "Collection render failed for #{f.size} document(s):\n" +
+        f.join("\n"),
+      )
     end
 
     def output_folder(opts)

data/lib/metanorma/collection/filelookup/filelookup.rb

@@ -3,7 +3,6 @@ require "htmlentities"
 require "metanorma-utils"
 require "marcel"
 require_relative "filelookup_sectionsplit"
-require_relative "base"
 require_relative "utils"
 
 module Metanorma
@@ -57,17 +56,18 @@ module Metanorma
         entry = file_entry(manifest, k, idx) or return
         bibdata_process(entry, i)
         bibitem_process(entry)
-        @files[key(i)] = entry
+        @files[entry_key(i)] = entry
       end
 
       def read_file_idents(manifest)
         id = manifest.identifier
-        sanitised_id = key(@isodoc.docid_prefix("", manifest.identifier.dup))
+        sanitised_id =
+          entry_key(@isodoc.docid_prefix("", manifest.identifier.dup))
         #       if manifest.bibdata and # NO, DO NOT FISH FOR THE GENUINE IDENTIFIER IN BIBDATA
         #         d = manifest.bibdata.docidentifier.detect { |x| x.primary } ||
         #           manifest.bibdata.docidentifier.first
         #         k = d.id
-        #         i = key(@isodoc.docid_prefix(d.type, d.id.dup))
+        #         i = entry_key(@isodoc.docid_prefix(d.type, d.id.dup))
         #       end
         [id, sanitised_id]
       end
@@ -158,24 +158,10 @@ module Metanorma
         end
       end
 
-      # Substitute special strings in filename patterns
-      # @param pattern [String] filename pattern with placeholders
-      # @param options [Hash] substitution values
-      # @option options [Integer] :document_num document index
-      # @option options [String] :basename filename without extension
-      # @option options [String] :basename_legacy full filename with extension
-      # @option options [Integer] :sectionsplit_num sectionsplit index
+      # Substitute special strings in filename patterns.
+      # See Util.substitute_filename_pattern.
       def substitute_filename_pattern(pattern, options = {})
-        pattern or return pattern
-        result = pattern.dup
-        options[:document_num] and
-          result.gsub!(/\{document-num\}/, options[:document_num].to_s)
-        result.gsub!(/\{basename\}/, options[:basename]) if options[:basename]
-        options[:basename_legacy] and
-          result.gsub!(/\{basename_legacy\}/, options[:basename_legacy])
-        options[:sectionsplit_num] and
-          result.gsub!(/\{sectionsplit-num\}/, options[:sectionsplit_num].to_s)
-        result
+        Util.substitute_filename_pattern(pattern, options)
       end
 
       # TODO make the output file location reflect source location universally,

data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb

@@ -18,7 +18,6 @@ module Metanorma
         # Save the original out_path before it gets modified
         original_out_path = @files[key][:out_path]
         s, sectionsplit_manifest = sectionsplit(key)
-        # section_split_instance_threads(s, manifest, key)
         s.each_with_index do |f1, i|
           add_section_split_instance(f1, manifest, key, i)
         end
@@ -29,18 +28,6 @@ module Metanorma
         original_out_path
       end
 
-      def section_split_instance_threads(s, manifest, key)
-        @mutex = Mutex.new
-        pool = Concurrent::FixedThreadPool.new(4)
-        s.each_with_index do |f1, i|
-          pool.post do
-            add_section_split_instance(f1, manifest, key, i)
-          end
-        end
-        pool.shutdown
-        pool.wait_for_termination
-      end
-
       def cleanup_section_split_instance(key, manifest, original_out_path)
         # Delete the sectionsplit index.html from source directory after it's copied to output
         @files_to_delete << manifest["#{key}:index.html"][:ref]
@@ -54,6 +41,14 @@ module Metanorma
         end
         # @files[key].delete(:ids).delete(:anchors)
         @files[key][:indirect_key] = @sectionsplit.key
+        # sectionsplit is HTML-only; hand the parent its whole Presentation XML
+        # so the collection PDF/presentation concatenation inlines the document
+        # intact (raw_file) instead of degrading to a bibdata-only, namespace-
+        # less stub. file_compile early-returns for sectionsplit, so this
+        # :outputs entry is what concatenate1 consumes.
+        # https://github.com/metanorma/iso-10303/issues/208
+        @files[key][:outputs] =
+          { presentation: @sectionsplit.whole_presentation_file }
       end
 
       def add_section_split_cover(manifest, sectionsplit_manifest, ident)
@@ -134,7 +129,7 @@ module Metanorma
         # file[:url] contains full path with directory for HTML output, but XML is basename only
         xml_basename = File.basename(file[:url])
         presfile = File.join(File.dirname(@files[key][:ref]), xml_basename)
-        newkey = key("#{key.strip} #{file[:title]}")
+        newkey = entry_key("#{key.strip} #{file[:title]}")
         xml = Nokogiri::XML(File.read(presfile), &:huge)
         [presfile, newkey, xml]
       end

data/lib/metanorma/collection/filelookup/utils.rb

@@ -1,56 +1,12 @@
 module Metanorma
   class Collection
     class FileLookup
-      # Also parse all ids in doc (including ones which won't be xref targets)
-      def read_ids(xml)
-        ret = {}
-        xml.traverse do |x|
-          x.text? and next
-          x["id"] and ret[x["id"]] = true
-        end
-        ret
-      end
-
-      # map locality type and label (e.g. "clause" "1") to id = anchor for
-      # a document
-      # Note: will only key clauses, which have unambiguous reference label in
-      # locality. Notes, examples etc with containers are just plunked against
-      # UUIDs, so that their IDs can at least be registered to be tracked
-      # as existing.
-      def read_anchors(xml)
-        xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n,
-                                  { locale: @locale })
-        xrefs.parse xml
-        xrefs.get.each_with_object({}) do |(k, v), ret|
-          read_anchors1(k, v, ret)
-        end
-      end
-
-      def read_anchors1(key, val, ret)
-        val[:type] ||= "clause"
-        ret[val[:type]] ||= {}
-        index = if val[:container] || val[:label].nil? || val[:label].empty?
-                  UUIDTools::UUID.random_create.to_s
-                else val[:label].gsub(%r{<[^<>]+>}, "")
-                end
-        ret[val[:type]][index] = key
-        v = val[:value] and ret[val[:type]][v.gsub(%r{<[^<>]+>}, "")] = key
-      end
-
       def anchors_lookup(anchors)
         anchors.values.each_with_object({}) do |v, m|
           v.each_value { |v1| m[v1] = true }
         end
       end
 
-      # return citation url for file
-      # @param doc [Boolean] I am a Metanorma document,
-      # so my URL should end with html or pdf or whatever
-      def url(ident, options)
-        data = get(ident)
-        data[:url] || targetfile(data, options)[1]
-      end
-
       # are references to the file to be linked to a file in the collection,
       # or externally? Determines whether file suffix anchors are to be used
       def url?(ident)
@@ -58,7 +14,10 @@ module Metanorma
         data[:url]
       end
 
-      def key(ident)
+      # Normalise an identifier to its @files hash key: decode entities, squeeze
+      # whitespace, and strip a leading "metanorma-collection " prefix. Distinct
+      # from Util::key, which does NOT strip that prefix.
+      def entry_key(ident)
         @c.decode(ident).gsub(/(\p{Zs})+/, " ")
           .sub(/^metanorma-collection /, "")
       end
@@ -68,13 +27,13 @@ module Metanorma
       end
 
       def get(ident, attr = nil)
-        if attr then @files[key(ident)][attr]
-        else @files[key(ident)]
+        if attr then @files[entry_key(ident)][attr]
+        else @files[entry_key(ident)]
         end
       end
 
       def set(ident, attr, value)
-        @files[key(ident)][attr] = value
+        @files[entry_key(ident)][attr] = value
       end
 
       def each

data/lib/metanorma/collection/log.rb

@@ -11,6 +11,13 @@ module Metanorma
       "METANORMA_3": { category: "Cross-References",
                        error: "<strong>** Unresolved reference to document %s from eref</strong>",
                        severity: 2 },
+      "METANORMA_4": { category: "Rendering",
+                       error: "Collection member %s: expected output " \
+                              "file not generated: %s",
+                       severity: 0 },
+      "METANORMA_5": { category: "Rendering",
+                       error: "Collection member %s: rendering error: %s",
+                       severity: 0 },
     }.freeze
     # rubocop:enable Naming/VariableNumber
   end

data/lib/metanorma/collection/manifest/manifest.rb

@@ -20,6 +20,12 @@ module Metanorma
         @config = manifest_postprocess(config)
       end
 
+      # Ordered pipeline of in-place tree-walks over the manifest; the order is
+      # load-bearing. expand_yaml must run early because it grafts nested
+      # `fileref` manifests into this tree, so every later pass sees the full
+      # entry set; bibdata sets @lang/@script used downstream; compile_adoc
+      # turns .adoc sources into .xml before filexist verifies them. Each pass
+      # recurses the manifest independently (see the architecture review plan).
       def manifest_postprocess(config)
         manifest_bibdata(config)
         manifest_expand_yaml(config, @dir)

data/lib/metanorma/collection/renderer/filelocation.rb

@@ -26,12 +26,16 @@ module Metanorma
         end
       end
 
-      # Move file to new output filename if specified
+      # Move file to new output filename if specified. A member that
+      # failed to render has nothing to move: keep the original name
+      # registered so output verification reports it as missing, rather
+      # than crashing the whole collection on the mv (#586).
       def handle_new_output_filename(output_fname, new_output_fname)
         return output_fname unless new_output_fname
 
-        FileUtils.mv(File.join(@outdir, output_fname),
-                     File.join(@outdir, new_output_fname))
+        src = File.join(@outdir, output_fname)
+        File.exist?(src) or return output_fname
+        FileUtils.mv(src, File.join(@outdir, new_output_fname))
         new_output_fname
       end
 

data/lib/metanorma/collection/renderer/fileparse.rb

@@ -14,24 +14,46 @@ module Metanorma
       # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to
       #   anchor to filename
       # @return [String] XML content
+      # Each pass below mutates the SAME Nokogiri tree in place; the inline
+      # comments state the post-condition each pass guarantees on exit. The
+      # contract between passes is "the tree is now in state X" -- there is no
+      # intermediate typed representation, so the ordering is load-bearing.
+      # `sso` truthy (sectionsplit_output) => `xml` is Presentation XML and the
+      # semantic-only passes are skipped; otherwise `xml` is Semantic XML.
+      # See the pipeline map in the collection architecture review plan.
       def update_xrefs(file, docid, internal_refs)
         xml, sso = update_xrefs_prep(file, docid)
-        @nested || sso or
+        # post: repository docidentifiers supplied on bibitems naming a
+        # collection file; `sso` resolved (Presentation vs Semantic).
+        @nested || sso || @reinflate or
           Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
                                                            @isodoc, sso)
+        # post (semantic only): intra-doc xref/eref turned into internal erefs;
+        # repo bibitems copied; indirect bibliography inserted.
         @ncnames = {}
         @nested or update_indirect_refs_to_docs(xml, docid, internal_refs, sso)
-        @files.add_document_suffix(docid, xml)
+        # post: bibitem[@type='internal'] anchors resolved to the containing
+        # collection document.
+        @reinflate or @files.add_document_suffix(docid, xml)
+        # post: every @id/@anchor namespaced with this doc's NCName suffix, so
+        # concatenated documents do not collide.
         @nested or update_sectionsplit_refs_to_docs(xml, docid, internal_refs,
                                                     sso)
+        # post: erefs targeting a sectionsplit document point at the specific
+        # split section file that holds the anchor.
         update_direct_refs_to_docs(xml, docid, sso)
+        # post: repo(current-metanorma-collection/x) bibitems replaced in situ
+        # with resolved bibdata + citation URI; their erefs become hyperlinks.
         ::Metanorma::Collection::Util::hide_refs(xml)
+        # post: references containers with only hidden bibitems flagged hidden.
         sso and eref2link(xml, sso)
-        @nested or svgmap_resolve(xml, docid, sso)
+        @nested || @reinflate or svgmap_resolve(xml, docid, sso)
         xml.to_xml
       end
 
-      ## sso files are Presentation XML; otherwise, Semantic XML
+      ## sso (sectionsplit_output) truthy => Presentation XML, else Semantic.
+      ## This is the single discriminator that decides which grammar `xml`
+      ## carries and therefore which passes in update_xrefs run.
       def update_xrefs_prep(file, docid)
         docxml = file.is_a?(String) ? Nokogiri::XML(file, &:huge) : file
         supply_repo_ids(docxml)
@@ -121,8 +143,13 @@ presxml)
       # Do not do this if this is a sectionsplit collection or a nested manifest.
       # Return false if bibitem is not to be further processed
       def strip_unresolved_repo_erefs(_document_id, bib_docid, erefs, bibitem)
-        %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text) and
+        if %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text)
+          # Isolated build: keep the cross-document ref as a stub (its bibitemid
+          # and repository docidentifier intact) for a later reinflation pass to
+          # relink, rather than resolving it now against an absent target.
+          @preserve_unresolved and return false
           return true
+        end
         @nested and return false
         erefs[bibitem["id"]]&.each { |x| x.parent and strip_eref(x) }
         false