metanorma 2.1.4 → 2.5.4

data/lib/metanorma/collection/manifest/manifest.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "../../util/util"
+require "metanorma-core"
 
 module Metanorma
   class Collection
@@ -14,17 +14,22 @@ module Metanorma
       # @param docref [Array<Hash{String=>String}>]
       # @param manifest [Array<Metanorma::Collection::Manifest>]
       def initialize(config, collection, dir)
-        #require "debug"; binding.b
         @collection = collection
         @dir = dir
         @disambig = ::Metanorma::Collection::Util::DisambigFiles.new
         @config = manifest_postprocess(config)
       end
 
+      # Ordered pipeline of in-place tree-walks over the manifest; the order is
+      # load-bearing. expand_yaml must run early because it grafts nested
+      # `fileref` manifests into this tree, so every later pass sees the full
+      # entry set; bibdata sets @lang/@script used downstream; compile_adoc
+      # turns .adoc sources into .xml before filexist verifies them. Each pass
+      # recurses the manifest independently (see the architecture review plan).
       def manifest_postprocess(config)
-        #require "debug"; binding.b
         manifest_bibdata(config)
         manifest_expand_yaml(config, @dir)
+        manifest_output_filenames(config)
         manifest_compile_adoc(config)
         manifest_filexist(config)
         manifest_sectionsplit(config)
@@ -40,6 +45,17 @@ module Metanorma
 
       GUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
 
+      def manifest_output_filenames(config, parent = nil)
+        config.output_filename ||=
+          parent&.output_filename # || "{basename}.{document-num}"
+        config.sectionsplit_filename ||= parent&.sectionsplit_filename ||
+          "{basename_legacy}.{sectionsplit-num}"
+        # "{basename}-{document-num}.{sectionsplit-num}"
+        Array(config.entry).each do |f|
+          manifest_output_filenames(f, config)
+        end
+      end
+
       def manifest_identifier(config)
         no_id = populate_id_from_doc(config)
         config.identifier =
@@ -96,7 +112,6 @@ module Metanorma
 
       def manifest_filexist(config)
         if config.file
-        #require "debug"; binding.b
           file = @collection.class.resolve_fileref(@dir, config.file)
           @collection.class.check_file_existence(file)
           config.file = Pathname.new(file).relative_path_from(Pathname.new(@dir))
@@ -111,7 +126,6 @@ module Metanorma
           currdir = dir
           /\.ya?ml$/.match?(e.file) and
             currdir = manifest_expand_yaml_entry(e, dir)
-          #require "debug"; binding.b
           manifest_expand_yaml(e, currdir)
         end
         config
@@ -122,10 +136,8 @@ module Metanorma
         currdir = File.dirname(f)
         @collection.class.check_file_existence(f)
         entry.file = nil
-        #require 'debug'; binding.b
         entry.entry = ::Metanorma::Collection::Config::Config.from_yaml(File.read(f)).manifest
         if currdir != dir
-          #require "debug"; binding.b
           prefix = Pathname.new(currdir).relative_path_from(Pathname.new(dir))
           update_filepaths(entry.entry, prefix.to_s)
         end
@@ -133,10 +145,9 @@ module Metanorma
       end
 
       def update_filepaths(entry, prefix)
-        #require "debug"; binding.b
         entry.file && !(Pathname.new entry.file).absolute? and
           entry.file = File.join(prefix, entry.file)
-        entry.entry.each do |f|
+        entry&.entry&.each do |f|
           update_filepaths(f, prefix)
         end
       end
@@ -170,18 +181,20 @@ module Metanorma
         f = (Pathname.new file).absolute? ? file : File.join(@dir, file)
         File.exist?(f) or raise AdocFileNotFoundException.new "#{f} not found!"
         compile_adoc_file?(file) or return
-        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...", :info)
+        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...", :warning)
         ::Metanorma::Compile.new
           .compile(f, agree_to_terms: true, install_fonts: false,
                       extension_keys: [:xml])
-        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...done!", :info)
+        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...done!",
+                              :warning)
       end
 
       def compile_adoc_file?(file)
+        !File.exist?(file.sub(/\.adoc$/, ".xml")) and return true
         @collection.directives.detect do |d|
-          d.key == "recompile-xml"
-        end and return true
-        !File.exist?(file.sub(/\.adoc$/, ".xml"))
+          d.key == "recompile-xml" && d.value == "false"
+        end and return false
+        true
       end
 
       def documents(mnf = @config)

data/lib/metanorma/collection/multilingual/multilingual.rb

@@ -617,7 +617,7 @@ module Metanorma
       end
 
       def htmlconv
-        x = Asciidoctor.load nil, backend: @flavor
+        x = Asciidoctor.load nil, backend: Util::taste2flavor(@flavor)
         x.converter.html_converter(@converter_opt)
       end
 

data/lib/metanorma/collection/renderer/filelocation.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Metanorma
+  class Collection
+    class Renderer
+      def file_compile_formats(filename, identifier, opts)
+        f = @files.get(identifier, :outputs)
+        format = opts[:extension_keys]
+        concatenate_presentation?({ format: }) and format << :presentation
+        format.each do |e|
+          e == :pdf and output_filename = opts[:pdffile]
+          file_compile_format(filename, identifier, e, f, output_filename)
+        end
+        @files.set(identifier, :outputs, f)
+      end
+
+      # Generate output filename with correct extension
+      def output_filename_with_extension(fname, format)
+        ext = @compile.processor.output_formats[format]
+        fname_base = File.basename(fname)
+        # ONLY replace .xml or .html extensions, NEVER remove section numbers
+        if /\.(xml|html)$/.match?(fname_base)
+          fname_base.sub(/\.(xml|html)$/, ".#{ext}")
+        else
+          "#{fname_base}.#{ext}"
+        end
+      end
+
+      # Move file to new output filename if specified. A member that
+      # failed to render has nothing to move: keep the original name
+      # registered so output verification reports it as missing, rather
+      # than crashing the whole collection on the mv (#586).
+      def handle_new_output_filename(output_fname, new_output_fname)
+        return output_fname unless new_output_fname
+
+        src = File.join(@outdir, output_fname)
+        File.exist?(src) or return output_fname
+        FileUtils.mv(src, File.join(@outdir, new_output_fname))
+        new_output_fname
+      end
+
+      # Apply custom filename pattern with substitutions
+      def apply_custom_filename_pattern(ident, custom_fname, output_fname,
+format)
+        idx = @files.get(ident, :idx)
+        original_file = @files.get(ident, :ref)
+        basename = File.basename(original_file, ".*")
+        basename_legacy = File.basename(original_file)
+        custom_fname = @files.substitute_filename_pattern(
+          custom_fname, document_num: idx,
+                        basename: basename, basename_legacy: basename_legacy
+        )
+        if File.dirname(custom_fname) == "."
+          output_fname
+        else
+          preserve_output_dir_structure(custom_fname, output_fname,
+                                        format, explicit_custom: true)
+        end
+      end
+
+      # Apply directory preservation for files with directory structure
+      def apply_directory_preservation(fname, output_fname, format)
+        fname_dir = File.dirname(fname)
+        output_fname = File.join(fname_dir, output_fname)
+        preserve_output_dir_structure(fname, output_fname, format)
+      end
+
+      # Compile single format for a file and update outputs hash
+      def file_compile_format(fname, ident, format, outputs, new_output_fname)
+        ext = @compile.processor.output_formats[format]
+        output_fname = output_filename_with_extension(fname, format)
+        output_fname = handle_new_output_filename(output_fname,
+                                                  new_output_fname)
+        if !new_output_fname && (custom = @files.preserve_directory_structure?(ident))
+          output_fname = apply_custom_filename_pattern(ident, custom,
+                                                       output_fname, format)
+        elsif !new_output_fname && File.dirname(fname) != "."
+          output_fname = apply_directory_preservation(fname, output_fname,
+                                                      format)
+        end
+        should_skip = /html$/.match?(ext) && @files.get(ident, :sectionsplit)
+        should_skip or outputs[format] = File.join(@outdir, output_fname)
+      end
+
+      # Determine if file should be moved to subdirectory
+      def should_move_to_subdirectory?(ext, output_basename, explicit_custom)
+        if explicit_custom
+          # For explicit output-filename: move html, doc, pdf, presentation.xml
+          ext.end_with?("html") || ext == "doc" || ext == "pdf" ||
+            output_basename.end_with?(".presentation.xml")
+        else
+          # Default: only HTML and presentation files
+          ext.end_with?("html") ||
+            output_basename.end_with?(".html", ".presentation.xml")
+        end
+      end
+
+      # Move file from root to subdirectory
+      def move_file_to_subdirectory(fname_dir, output_basename)
+        output_with_dir = File.join(fname_dir, output_basename)
+        output_dest = File.join(@outdir, output_with_dir)
+        output_src = File.join(@outdir, output_basename)
+        return nil unless File.exist?(output_src) && output_src != output_dest
+
+        FileUtils.mkdir_p(File.dirname(output_dest))
+        FileUtils.mv(output_src, output_dest)
+        output_with_dir
+      end
+
+      # Handle case where destination already exists
+      def handle_existing_destination(output_with_dir, output_src)
+        output_dest = File.join(@outdir, output_with_dir)
+        return nil unless File.exist?(output_dest)
+        return output_with_dir unless File.exist?(output_src)
+
+        FileUtils.rm_f(output_src)
+        err_src = output_src.sub(/\.html$/, ".err.html")
+        FileUtils.rm_f(err_src) if File.exist?(err_src)
+        output_with_dir
+      end
+
+      # Generate custom basename with extension
+      def custom_basename_with_extension(fname, output_basename)
+        custom_basename = File.basename(fname, ".*")
+        ext = File.extname(output_basename)
+        "#{custom_basename}#{ext}"
+      end
+
+      # Preserve directory structure from input filename in output
+      def preserve_output_dir_structure(fname, output_fname, format = nil,
+explicit_custom: false)
+        fname_dir = File.dirname(fname)
+        return output_fname if fname_dir == "."
+
+        output_basename = File.basename(output_fname)
+        ext = format ? @compile.processor.output_formats[format].to_s : ""
+        should_move = should_move_to_subdirectory?(ext, output_basename,
+                                                   explicit_custom)
+        if should_move
+          result = move_file_to_subdirectory(fname_dir, output_basename)
+          return result if result
+
+          handle_existing_destination(File.join(fname_dir, output_basename),
+                                      File.join(@outdir,
+                                                output_basename)) || output_fname
+        elsif explicit_custom
+          custom_basename_with_extension(fname, output_basename)
+        else
+          output_basename
+        end
+      end
+
+      def copy_file_to_dest(identifier)
+        out = Pathname.new(@files.get(identifier, :out_path)).cleanpath
+        out.absolute? and
+          out = out.relative_path_from(File.expand_path(FileUtils.pwd))
+        dest = File.join(@outdir,
+                         @disambig.source2dest_filename(out.to_s,
+                                                        preserve_dirs: true))
+        FileUtils.mkdir_p(File.dirname(dest))
+        source = @files.get(identifier, :ref)
+        source != dest and FileUtils.cp_r source, dest, remove_destination: true
+      end
+    end
+  end
+end

data/lib/metanorma/collection/renderer/fileparse.rb

@@ -4,7 +4,7 @@ module Metanorma
       # Resolves references to other files in the collection. Three routines:
       # 1. Eref to a document that has been split into multiple documents
       # (sectionsplit) are resolved to direct eref to the split document
-      # 2. Indirect erefs to a file anchor in an unknwon file in the collection
+      # 2. Indirect erefs to a file anchor in an unknown file in the collection
       # (bibitem[@type = 'internal'] ) are resolved to direct eref to the
       # containing document
       # 3. Direct erefs to other files in collection
@@ -14,25 +14,46 @@ module Metanorma
       # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to
       #   anchor to filename
       # @return [String] XML content
+      # Each pass below mutates the SAME Nokogiri tree in place; the inline
+      # comments state the post-condition each pass guarantees on exit. The
+      # contract between passes is "the tree is now in state X" -- there is no
+      # intermediate typed representation, so the ordering is load-bearing.
+      # `sso` truthy (sectionsplit_output) => `xml` is Presentation XML and the
+      # semantic-only passes are skipped; otherwise `xml` is Semantic XML.
+      # See the pipeline map in the collection architecture review plan.
       def update_xrefs(file, docid, internal_refs)
         xml, sso = update_xrefs_prep(file, docid)
-        #require "debug"; binding.b if /This document is also unrelated/.match?(xml.to_xml)
-        #warn (/fmt-title/.match?(xml.to_xml) ? "*** PRESENTATION" : "*** SEMANTIC")
-        @nested || sso or
+        # post: repository docidentifiers supplied on bibitems naming a
+        # collection file; `sso` resolved (Presentation vs Semantic).
+        @nested || sso || @reinflate or
           Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
                                                            @isodoc, sso)
+        # post (semantic only): intra-doc xref/eref turned into internal erefs;
+        # repo bibitems copied; indirect bibliography inserted.
         @ncnames = {}
         @nested or update_indirect_refs_to_docs(xml, docid, internal_refs, sso)
-        @files.add_document_suffix(docid, xml)
-        @nested or update_sectionsplit_refs_to_docs(xml, internal_refs, sso)
+        # post: bibitem[@type='internal'] anchors resolved to the containing
+        # collection document.
+        @reinflate or @files.add_document_suffix(docid, xml)
+        # post: every @id/@anchor namespaced with this doc's NCName suffix, so
+        # concatenated documents do not collide.
+        @nested or update_sectionsplit_refs_to_docs(xml, docid, internal_refs,
+                                                    sso)
+        # post: erefs targeting a sectionsplit document point at the specific
+        # split section file that holds the anchor.
         update_direct_refs_to_docs(xml, docid, sso)
-        hide_refs(xml)
+        # post: repo(current-metanorma-collection/x) bibitems replaced in situ
+        # with resolved bibdata + citation URI; their erefs become hyperlinks.
+        ::Metanorma::Collection::Util::hide_refs(xml)
+        # post: references containers with only hidden bibitems flagged hidden.
         sso and eref2link(xml, sso)
-        @nested or svgmap_resolve(xml, docid, sso)
+        @nested || @reinflate or svgmap_resolve(xml, docid, sso)
         xml.to_xml
       end
 
-      ## sso files are Presentation XML; otherwise, Semantic XML
+      ## sso (sectionsplit_output) truthy => Presentation XML, else Semantic.
+      ## This is the single discriminator that decides which grammar `xml`
+      ## carries and therefore which passes in update_xrefs run.
       def update_xrefs_prep(file, docid)
         docxml = file.is_a?(String) ? Nokogiri::XML(file, &:huge) : file
         supply_repo_ids(docxml)
@@ -40,7 +61,8 @@ module Metanorma
         [docxml, sso]
       end
 
-      def update_sectionsplit_refs_to_docs(docxml, internal_refs, presxml)
+      def update_sectionsplit_refs_to_docs(docxml, docid, internal_refs,
+presxml)
         Util::gather_citeases(docxml, presxml).each do |k, v|
           (@files.get(k) && @files.get(k, :sectionsplit)) or next
           opts = { key: @files.get(k, :indirect_key),
@@ -49,7 +71,7 @@ module Metanorma
           refs = v.each_with_object({}) do |eref, m|
             update_sectionsplit_eref_to_doc(eref, internal_refs, m, opts)
           end
-          add_hidden_bibliography(docxml, refs)
+          add_hidden_bibliography(docxml, refs, docid)
         end
       end
 
@@ -65,8 +87,9 @@ module Metanorma
         doclist
       end
 
-      BIBITEM_NOT_REPO_XPATH = "//bibitem[not(ancestor::bibitem)]" \
-        "[not(./docidentifier[@type = 'repository'])]".freeze
+      BIBITEM_NOT_REPO_XPATH = <<~XPATH.strip
+        //bibitem[not(ancestor::bibitem)][not(ancestor::bibdata)][not(./docidentifier[@type = 'repository'])]
+      XPATH
 
       def supply_repo_ids(doc)
         doc.xpath(ns(BIBITEM_NOT_REPO_XPATH)).each do |b|
@@ -84,10 +107,11 @@ module Metanorma
       # Any erefs to that bibitem id are replaced with relative URL
       # Preferably with anchor, and is a job to realise dynamic lookup
       # of localities.
-      def update_direct_refs_to_docs(docxml, identifier, presxml)
+      def update_direct_refs_to_docs(xml, identifier, presxml)
         erefs, erefs_no_anchor, anchors, erefs1 =
-          update_direct_refs_to_docs_prep(docxml, presxml)
-        docxml.xpath(ns("//bibitem")).each do |b|
+          update_direct_refs_to_docs_prep(xml, presxml)
+        x = xml.xpath(ns("//bibitem")) - xml.xpath(ns("//bibdata//bibitem"))
+        x.each do |b|
           docid = b.at(ns("./docidentifier[@type = 'repository']")) or next
           strip_unresolved_repo_erefs(identifier, docid, erefs1, b) or next
           update_bibitem(b, identifier)
@@ -106,10 +130,10 @@ module Metanorma
         erefs.each do |k, v|
           v.each do |e|
             if loc = e.at(".//#{ANCHOR_XPATH}") then anchors[k] << loc
-            else no_anchor[k] << e end
+            else no_anchor[k] << e
+            end
           end
         end
-        #require "debug"; binding.b
         [erefs, no_anchor, anchors, Util::gather_bibitemids(docxml, presxml)]
       end
 
@@ -119,8 +143,13 @@ module Metanorma
       # Do not do this if this is a sectionsplit collection or a nested manifest.
       # Return false if bibitem is not to be further processed
       def strip_unresolved_repo_erefs(_document_id, bib_docid, erefs, bibitem)
-        %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text) and
+        if %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text)
+          # Isolated build: keep the cross-document ref as a stub (its bibitemid
+          # and repository docidentifier intact) for a later reinflation pass to
+          # relink, rather than resolving it now against an absent target.
+          @preserve_unresolved and return false
           return true
+        end
         @nested and return false
         erefs[bibitem["id"]]&.each { |x| x.parent and strip_eref(x) }
         false
@@ -128,8 +157,9 @@ module Metanorma
 
       # Resolve erefs to a container of ids in another doc,
       # to an anchor eref (direct link)
-      def update_indirect_refs_to_docs(docxml, _docidentifier, internal_refs, presxml)
-        bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml, presxml)
+      def update_indirect_refs_to_docs(docxml, _docid, internal_refs, presxml)
+        bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml,
+                                                                        presxml)
         internal_refs.each do |schema, ids|
           add_suffix = doc_suffix && doc_type && doc_type != schema
           ids.each do |id, file|
@@ -149,37 +179,15 @@ module Metanorma
          docxml.root["document_suffix"], docxml.root["type"], {}]
       end
 
-      # KILL
-      def indirect_ref_key(schema, id, doc_suffix, doc_type)
-        /^#{schema}_/.match?(id) and return id
-        key = [schema, id, doc_suffix, doc_type].join("::")
-        x = @indirect_keys[key] and return x
-        ret = "#{schema}_#{id}"
-        doc_suffix && doc_type && doc_type != schema and
-          ret = "#{ret}_#{doc_suffix}"
-        @indirect_keys[key] = ret
-        ret
-      end
-
       def indirect_ref_key(schema, id, doc_suffix, add_suffix)
         /^#{schema}_/.match?(id) and return id
-        #key = "#{schema}_#{id}"
         x = @indirect_keys.dig(schema, id) and return x
         @indirect_keys[schema] ||= {}
         @indirect_keys[schema][id] = if add_suffix
-                                  "#{schema}_#{id}_#{doc_suffix}"
-                                else
-                                   "#{schema}_#{id}"
-                                end
-      end
-
-      # KILL
-      def indirect_ref_keyx(schema, id, doc_suffix, doc_type)
-        /^#{schema}_/.match?(id) and return id
-        ret = "#{schema}_#{id}"
-        doc_suffix && doc_type && doc_type != schema and
-          ret = "#{ret}_#{doc_suffix}"
-        ret
+                                       "#{schema}_#{id}_#{doc_suffix}"
+                                     else
+                                       "#{schema}_#{id}"
+                                     end
       end
 
       def update_indirect_refs_to_docs1(filec, key, file, bibitems, erefs)
@@ -196,13 +204,10 @@ module Metanorma
         parentid and file = "#{parentid}_#{file}"
         existing = a.text
         anchor = if url then existing
-                   else 
-                     #suffix_anchor_indirect(existing, suffix)
-                     #k = "#{existing}_#{file}"
-        #@ncnames[k] ||= Metanorma::Utils::to_ncname(k)
-        @indirect_keys[existing] ||= {}
-        @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
-                   end
+                 else
+                   @indirect_keys[existing] ||= {}
+                   @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
+                 end
         @updated_anchors[existing] or a.children = anchor
         @updated_anchors[anchor] = true
       end
@@ -254,7 +259,7 @@ module Metanorma
         type = "clause" if type == "annex"
         ref = ins.at(ns("./locality/referenceFrom"))&.text
         a = @files.get(docid, :anchors).dig(type, ref) or return
-        ins << "<locality type='anchor'><referenceFrom>#{a.sub(/^_/, '')}" \
+        ins << "<locality type='anchor'><referenceFrom>#{a}" \
           "</referenceFrom></locality>"
       end
     end