metanorma 2.0.0 → 2.5.5

data/lib/metanorma/collection/renderer/filelocation.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Metanorma
+  class Collection
+    class Renderer
+      def file_compile_formats(filename, identifier, opts)
+        f = @files.get(identifier, :outputs)
+        format = opts[:extension_keys]
+        concatenate_presentation?({ format: }) and format << :presentation
+        format.each do |e|
+          e == :pdf and output_filename = opts[:pdffile]
+          file_compile_format(filename, identifier, e, f, output_filename)
+        end
+        @files.set(identifier, :outputs, f)
+      end
+
+      # Generate output filename with correct extension
+      def output_filename_with_extension(fname, format)
+        ext = @compile.processor.output_formats[format]
+        fname_base = File.basename(fname)
+        # ONLY replace .xml or .html extensions, NEVER remove section numbers
+        if /\.(xml|html)$/.match?(fname_base)
+          fname_base.sub(/\.(xml|html)$/, ".#{ext}")
+        else
+          "#{fname_base}.#{ext}"
+        end
+      end
+
+      # Move file to new output filename if specified. A member that
+      # failed to render has nothing to move: keep the original name
+      # registered so output verification reports it as missing, rather
+      # than crashing the whole collection on the mv (#586).
+      def handle_new_output_filename(output_fname, new_output_fname)
+        return output_fname unless new_output_fname
+
+        src = File.join(@outdir, output_fname)
+        File.exist?(src) or return output_fname
+        FileUtils.mv(src, File.join(@outdir, new_output_fname))
+        new_output_fname
+      end
+
+      # Apply custom filename pattern with substitutions
+      def apply_custom_filename_pattern(ident, custom_fname, output_fname,
+format)
+        idx = @files.get(ident, :idx)
+        original_file = @files.get(ident, :ref)
+        basename = File.basename(original_file, ".*")
+        basename_legacy = File.basename(original_file)
+        custom_fname = @files.substitute_filename_pattern(
+          custom_fname, document_num: idx,
+                        basename: basename, basename_legacy: basename_legacy
+        )
+        if File.dirname(custom_fname) == "."
+          output_fname
+        else
+          preserve_output_dir_structure(custom_fname, output_fname,
+                                        format, explicit_custom: true)
+        end
+      end
+
+      # Apply directory preservation for files with directory structure
+      def apply_directory_preservation(fname, output_fname, format)
+        fname_dir = File.dirname(fname)
+        output_fname = File.join(fname_dir, output_fname)
+        preserve_output_dir_structure(fname, output_fname, format)
+      end
+
+      # Compile single format for a file and update outputs hash
+      def file_compile_format(fname, ident, format, outputs, new_output_fname)
+        ext = @compile.processor.output_formats[format]
+        output_fname = output_filename_with_extension(fname, format)
+        output_fname = handle_new_output_filename(output_fname,
+                                                  new_output_fname)
+        if !new_output_fname && (custom = @files.preserve_directory_structure?(ident))
+          output_fname = apply_custom_filename_pattern(ident, custom,
+                                                       output_fname, format)
+        elsif !new_output_fname && File.dirname(fname) != "."
+          output_fname = apply_directory_preservation(fname, output_fname,
+                                                      format)
+        end
+        should_skip = /html$/.match?(ext) && @files.get(ident, :sectionsplit)
+        should_skip or outputs[format] = File.join(@outdir, output_fname)
+      end
+
+      # Determine if file should be moved to subdirectory
+      def should_move_to_subdirectory?(ext, output_basename, explicit_custom)
+        if explicit_custom
+          # For explicit output-filename: move html, doc, pdf, presentation.xml
+          ext.end_with?("html") || ext == "doc" || ext == "pdf" ||
+            output_basename.end_with?(".presentation.xml")
+        else
+          # Default: only HTML and presentation files
+          ext.end_with?("html") ||
+            output_basename.end_with?(".html", ".presentation.xml")
+        end
+      end
+
+      # Move file from root to subdirectory
+      def move_file_to_subdirectory(fname_dir, output_basename)
+        output_with_dir = File.join(fname_dir, output_basename)
+        output_dest = File.join(@outdir, output_with_dir)
+        output_src = File.join(@outdir, output_basename)
+        return nil unless File.exist?(output_src) && output_src != output_dest
+
+        FileUtils.mkdir_p(File.dirname(output_dest))
+        FileUtils.mv(output_src, output_dest)
+        output_with_dir
+      end
+
+      # Handle case where destination already exists
+      def handle_existing_destination(output_with_dir, output_src)
+        output_dest = File.join(@outdir, output_with_dir)
+        return nil unless File.exist?(output_dest)
+        return output_with_dir unless File.exist?(output_src)
+
+        FileUtils.rm_f(output_src)
+        err_src = output_src.sub(/\.html$/, ".err.html")
+        FileUtils.rm_f(err_src) if File.exist?(err_src)
+        output_with_dir
+      end
+
+      # Generate custom basename with extension
+      def custom_basename_with_extension(fname, output_basename)
+        custom_basename = File.basename(fname, ".*")
+        ext = File.extname(output_basename)
+        "#{custom_basename}#{ext}"
+      end
+
+      # Preserve directory structure from input filename in output
+      def preserve_output_dir_structure(fname, output_fname, format = nil,
+explicit_custom: false)
+        fname_dir = File.dirname(fname)
+        return output_fname if fname_dir == "."
+
+        output_basename = File.basename(output_fname)
+        ext = format ? @compile.processor.output_formats[format].to_s : ""
+        should_move = should_move_to_subdirectory?(ext, output_basename,
+                                                   explicit_custom)
+        if should_move
+          result = move_file_to_subdirectory(fname_dir, output_basename)
+          return result if result
+
+          handle_existing_destination(File.join(fname_dir, output_basename),
+                                      File.join(@outdir,
+                                                output_basename)) || output_fname
+        elsif explicit_custom
+          custom_basename_with_extension(fname, output_basename)
+        else
+          output_basename
+        end
+      end
+
+      def copy_file_to_dest(identifier)
+        out = Pathname.new(@files.get(identifier, :out_path)).cleanpath
+        out.absolute? and
+          out = out.relative_path_from(File.expand_path(FileUtils.pwd))
+        dest = File.join(@outdir,
+                         @disambig.source2dest_filename(out.to_s,
+                                                        preserve_dirs: true))
+        FileUtils.mkdir_p(File.dirname(dest))
+        source = @files.get(identifier, :ref)
+        source != dest and FileUtils.cp_r source, dest, remove_destination: true
+      end
+    end
+  end
+end

data/lib/metanorma/collection/renderer/fileparse.rb

@@ -4,7 +4,7 @@ module Metanorma
       # Resolves references to other files in the collection. Three routines:
       # 1. Eref to a document that has been split into multiple documents
       # (sectionsplit) are resolved to direct eref to the split document
-      # 2. Indirect erefs to a file anchor in an unknwon file in the collection
+      # 2. Indirect erefs to a file anchor in an unknown file in the collection
       # (bibitem[@type = 'internal'] ) are resolved to direct eref to the
       # containing document
       # 3. Direct erefs to other files in collection
@@ -14,21 +14,46 @@ module Metanorma
       # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to
       #   anchor to filename
       # @return [String] XML content
+      # Each pass below mutates the SAME Nokogiri tree in place; the inline
+      # comments state the post-condition each pass guarantees on exit. The
+      # contract between passes is "the tree is now in state X" -- there is no
+      # intermediate typed representation, so the ordering is load-bearing.
+      # `sso` truthy (sectionsplit_output) => `xml` is Presentation XML and the
+      # semantic-only passes are skipped; otherwise `xml` is Semantic XML.
+      # See the pipeline map in the collection architecture review plan.
       def update_xrefs(file, docid, internal_refs)
         xml, sso = update_xrefs_prep(file, docid)
-        @nested || sso or
+        # post: repository docidentifiers supplied on bibitems naming a
+        # collection file; `sso` resolved (Presentation vs Semantic).
+        @nested || sso || @reinflate or
           Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
-                                                           @isodoc)
-        @nested or update_indirect_refs_to_docs(xml, docid, internal_refs)
-        @files.add_document_suffix(docid, xml)
-        @nested or update_sectionsplit_refs_to_docs(xml, internal_refs)
-        update_direct_refs_to_docs(xml, docid)
-        hide_refs(xml)
-        sso and eref2link(xml)
-        svgmap_resolve(xml, docid)
+                                                           @isodoc, sso)
+        # post (semantic only): intra-doc xref/eref turned into internal erefs;
+        # repo bibitems copied; indirect bibliography inserted.
+        @ncnames = {}
+        @nested or update_indirect_refs_to_docs(xml, docid, internal_refs, sso)
+        # post: bibitem[@type='internal'] anchors resolved to the containing
+        # collection document.
+        @reinflate or @files.add_document_suffix(docid, xml)
+        # post: every @id/@anchor namespaced with this doc's NCName suffix, so
+        # concatenated documents do not collide.
+        @nested or update_sectionsplit_refs_to_docs(xml, docid, internal_refs,
+                                                    sso)
+        # post: erefs targeting a sectionsplit document point at the specific
+        # split section file that holds the anchor.
+        update_direct_refs_to_docs(xml, docid, sso)
+        # post: repo(current-metanorma-collection/x) bibitems replaced in situ
+        # with resolved bibdata + citation URI; their erefs become hyperlinks.
+        ::Metanorma::Collection::Util::hide_refs(xml)
+        # post: references containers with only hidden bibitems flagged hidden.
+        sso and eref2link(xml, sso)
+        @nested || @reinflate or svgmap_resolve(xml, docid, sso)
         xml.to_xml
       end
 
+      ## sso (sectionsplit_output) truthy => Presentation XML, else Semantic.
+      ## This is the single discriminator that decides which grammar `xml`
+      ## carries and therefore which passes in update_xrefs run.
       def update_xrefs_prep(file, docid)
         docxml = file.is_a?(String) ? Nokogiri::XML(file, &:huge) : file
         supply_repo_ids(docxml)
@@ -36,8 +61,9 @@ module Metanorma
         [docxml, sso]
       end
 
-      def update_sectionsplit_refs_to_docs(docxml, internal_refs)
-        Util::gather_citeases(docxml).each do |k, v|
+      def update_sectionsplit_refs_to_docs(docxml, docid, internal_refs,
+presxml)
+        Util::gather_citeases(docxml, presxml).each do |k, v|
           (@files.get(k) && @files.get(k, :sectionsplit)) or next
           opts = { key: @files.get(k, :indirect_key),
                    source_suffix: docxml.root["document_suffix"],
@@ -45,13 +71,15 @@ module Metanorma
           refs = v.each_with_object({}) do |eref, m|
             update_sectionsplit_eref_to_doc(eref, internal_refs, m, opts)
           end
-          add_hidden_bibliography(docxml, refs)
+          add_hidden_bibliography(docxml, refs, docid)
         end
       end
 
+      ANCHOR_XPATH = "xmlns:locality[@type = 'anchor']/xmlns:referenceFrom"
+        .freeze
+
       def update_sectionsplit_eref_to_doc(eref, internal_refs, doclist, opts)
-        a = eref.at(ns("./localityStack/locality[@type = 'anchor']/" \
-                       "referenceFrom")) or return
+        a = eref.at("./xmlns:localityStack/#{ANCHOR_XPATH}") or return
         doc = internal_refs[opts[:key]]["#{a.text}_#{opts[:target_suffix]}"]
         bibitemid = Metanorma::Utils::to_ncname("#{doc}_#{opts[:source_suffix]}")
         eref["bibitemid"] = bibitemid
@@ -59,21 +87,9 @@ module Metanorma
         doclist
       end
 
-      def new_hidden_ref(xmldoc)
-        ins = xmldoc.at(ns("bibliography")) or
-          xmldoc.root << "<bibliography/>" and ins = xmldoc.at(ns("bibliography"))
-        ins.at(ns("./referenced[@hidden = 'true']")) or
-          ins.add_child("<references hidden='true' normative='false'/>").first
-      end
-
-      def eref2link(docxml)
-        isodoc = IsoDoc::PresentationXMLConvert.new({})
-        isodoc.bibitem_lookup(docxml)
-        isodoc.eref2link(docxml)
-      end
-
-      BIBITEM_NOT_REPO_XPATH = "//bibitem[not(ancestor::bibitem)]" \
-        "[not(./docidentifier[@type = 'repository'])]".freeze
+      BIBITEM_NOT_REPO_XPATH = <<~XPATH.strip
+        //bibitem[not(ancestor::bibitem)][not(ancestor::bibdata)][not(./docidentifier[@type = 'repository'])]
+      XPATH
 
       def supply_repo_ids(doc)
         doc.xpath(ns(BIBITEM_NOT_REPO_XPATH)).each do |b|
@@ -86,47 +102,39 @@ module Metanorma
         end
       end
 
-      def svgmap_resolve(docxml, docid)
-        ids = @files.get(docid, :ids)
-        dir = File.join(@dirname, File.dirname(@files.get(docid, :rel_path)))
-        docxml = datauri_encode(docxml, dir)
-        isodoc = IsoDoc::PresentationXMLConvert.new({})
-        isodoc.bibitem_lookup(docxml)
-        docxml.xpath(ns("//svgmap//eref")).each do |e|
-          svgmap_resolve1(e, isodoc, docxml, ids)
-        end
-        Vectory::SvgMapping.new(docxml, "").call
-        docxml.xpath(ns("//svgmap")).each { |s| isodoc.svgmap_extract(s) }
-      end
-
-      def svgmap_resolve1(eref, isodoc, _docxml, ids)
-        href = isodoc.eref_target(eref) or return
-        href == "##{eref['bibitemid']}" ||
-          (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
-        eref["target"] = href.strip
-        eref.name = "link"
-        eref.elements&.remove
-      end
-
       # repo(current-metanorma-collection/ISO 17301-1:2016)
       # replaced by bibdata of "ISO 17301-1:2016" in situ as bibitem.
       # Any erefs to that bibitem id are replaced with relative URL
       # Preferably with anchor, and is a job to realise dynamic lookup
       # of localities.
-      def update_direct_refs_to_docs(docxml, identifier)
-        erefs, erefs1 = update_direct_refs_to_docs_prep(docxml)
-        docxml.xpath(ns("//bibitem")).each do |b|
+      def update_direct_refs_to_docs(xml, identifier, presxml)
+        erefs, erefs_no_anchor, anchors, erefs1 =
+          update_direct_refs_to_docs_prep(xml, presxml)
+        x = xml.xpath(ns("//bibitem")) - xml.xpath(ns("//bibdata//bibitem"))
+        x.each do |b|
           docid = b.at(ns("./docidentifier[@type = 'repository']")) or next
           strip_unresolved_repo_erefs(identifier, docid, erefs1, b) or next
           update_bibitem(b, identifier)
           docid = docid_to_citeas(b) or next
-          erefs[docid] and update_anchors(b, docid, erefs[docid])
+          erefs[docid] and
+            update_anchors(b, docid, erefs[docid], erefs_no_anchor[docid],
+                           anchors[docid])
         end
       end
 
-      def update_direct_refs_to_docs_prep(docxml)
-        @ncnames = {}
-        [Util::gather_citeases(docxml), Util::gather_bibitemids(docxml)]
+      # Hash(docid) of arrays
+      def update_direct_refs_to_docs_prep(docxml, presxml)
+        erefs = Util::gather_citeases(docxml, presxml)
+        no_anchor = erefs.keys.each_with_object({}) { |k, m| m[k] = [] }
+        anchors = erefs.keys.each_with_object({}) { |k, m| m[k] = [] }
+        erefs.each do |k, v|
+          v.each do |e|
+            if loc = e.at(".//#{ANCHOR_XPATH}") then anchors[k] << loc
+            else no_anchor[k] << e
+            end
+          end
+        end
+        [erefs, no_anchor, anchors, Util::gather_bibitemids(docxml, presxml)]
       end
 
       # strip erefs if they are repository erefs, but do not point to a document
@@ -135,8 +143,13 @@ module Metanorma
       # Do not do this if this is a sectionsplit collection or a nested manifest.
       # Return false if bibitem is not to be further processed
       def strip_unresolved_repo_erefs(_document_id, bib_docid, erefs, bibitem)
-        %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text) and
+        if %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text)
+          # Isolated build: keep the cross-document ref as a stub (its bibitemid
+          # and repository docidentifier intact) for a later reinflation pass to
+          # relink, rather than resolving it now against an absent target.
+          @preserve_unresolved and return false
           return true
+        end
         @nested and return false
         erefs[bibitem["id"]]&.each { |x| x.parent and strip_eref(x) }
         false
@@ -144,90 +157,98 @@ module Metanorma
 
       # Resolve erefs to a container of ids in another doc,
       # to an anchor eref (direct link)
-      def update_indirect_refs_to_docs(docxml, _docidentifier, internal_refs)
-        bibitems, erefs = update_indirect_refs_to_docs_prep(docxml)
+      def update_indirect_refs_to_docs(docxml, _docid, internal_refs, presxml)
+        bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml,
+                                                                        presxml)
         internal_refs.each do |schema, ids|
+          add_suffix = doc_suffix && doc_type && doc_type != schema
           ids.each do |id, file|
-            k = indirect_ref_key(schema, id, docxml)
-            update_indirect_refs_to_docs1(docxml, k, file, bibitems, erefs)
+            f[file] ||= { url: @files.url?(file),
+                          parentid: @files.get(file) && @files.get(file,
+                                                                   :parentid) }
+            k = indirect_ref_key(schema, id, doc_suffix, add_suffix)
+            update_indirect_refs_to_docs1(f[file], k, file, bib, erefs)
           end
         end
       end
 
-      def update_indirect_refs_to_docs_prep(docxml)
+      def update_indirect_refs_prep(docxml, presxml)
         @updated_anchors = {}
-        [Util::gather_bibitems(docxml), Util::gather_bibitemids(docxml)]
+        @indirect_keys = {}
+        [Util::gather_bibitems(docxml), Util::gather_bibitemids(docxml, presxml),
+         docxml.root["document_suffix"], docxml.root["type"], {}]
       end
 
-      def indirect_ref_key(schema, id, docxml)
+      def indirect_ref_key(schema, id, doc_suffix, add_suffix)
         /^#{schema}_/.match?(id) and return id
-        ret = "#{schema}_#{id}"
-        suffix = docxml.root["document_suffix"]
-        (k = docxml.root["type"]) && k != schema && suffix and
-          ret = "#{ret}_#{suffix}"
-        ret
+        x = @indirect_keys.dig(schema, id) and return x
+        @indirect_keys[schema] ||= {}
+        @indirect_keys[schema][id] = if add_suffix
+                                       "#{schema}_#{id}_#{doc_suffix}"
+                                     else
+                                       "#{schema}_#{id}"
+                                     end
       end
 
-      def update_indirect_refs_to_docs1(_docxml, key, file, bibitems, erefs)
+      def update_indirect_refs_to_docs1(filec, key, file, bibitems, erefs)
         erefs[key]&.each do |e|
           e["citeas"] = file
-          update_indirect_refs_to_docs_anchor(e, file)
+          update_indirect_refs_to_docs_anchor(e, file, filec[:url],
+                                              filec[:parentid])
         end
         update_indirect_refs_to_docs_docid(bibitems[key], file)
       end
 
-      def update_indirect_refs_to_docs_anchor(eref, file)
-        a = eref.at(ns(".//locality[@type = 'anchor']/referenceFrom")) or return
-        suffix = file
-        @files.get(file) && p = @files.get(file, :parentid) and
-          suffix = "#{p}_#{suffix}"
+      def update_indirect_refs_to_docs_anchor(eref, file, url, parentid)
+        a = eref.at(".//#{ANCHOR_XPATH}") or return
+        parentid and file = "#{parentid}_#{file}"
         existing = a.text
-        anchor = existing
-        @files.url?(file) or
-          anchor = Metanorma::Utils::to_ncname("#{anchor}_#{suffix}")
+        anchor = if url then existing
+                 else
+                   @indirect_keys[existing] ||= {}
+                   @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
+                 end
         @updated_anchors[existing] or a.children = anchor
         @updated_anchors[anchor] = true
       end
 
-      def update_indirect_refs_to_docs_docid(bibitem, file)
-        docid = bibitem&.at(ns("./docidentifier[@type = 'repository']")) or
-          return
+      def update_indirect_refs_to_docs_docid(bib, file)
+        docid = bib&.at(ns("./docidentifier[@type = 'repository']")) or return
         docid.children = "current-metanorma-collection/#{file}"
         docid.previous =
           "<docidentifier type='metanorma-collection'>#{file}</docidentifier>"
       end
 
-      # update crossrefences to other documents, to include
-      # disambiguating document suffix on id
-      def update_anchors(bib, docid, erefs)
-        erefs.each do |e|
-          if @files.get(docid) then update_anchor_loc(bib, e, docid)
-          else
-            msg = "<strong>** Unresolved reference to document #{docid} " \
-                  "from eref</strong>"
-            e << msg
-            strip_eref(e)
-            @log&.add("Cross-References", e, msg)
-          end
+      # bottleneck
+      def update_anchors(bib, docid, erefs, erefs_no_anchor, erefs_anchors)
+        @files.get(docid) or error_anchor(erefs, docid)
+        has_anchors, url, ncn_docid = update_anchors_prep(docid)
+        erefs_no_anchor.each do |e|
+          update_anchor_create_loc(bib, e, docid)
+        end
+        !url && has_anchors or return
+        erefs_anchors.each do |e|
+          update_anchors1(docid, ncn_docid, e)
+        end
+      end
+
+      def update_anchors1(docid, ncn_docid, anchor)
+        @concat_anchors[anchor.text] ||= "#{ncn_docid}_#{anchor.text}"
+        if @files.get(docid).dig(:anchors_lookup, @concat_anchors[anchor.text])
+          anchor.content = @concat_anchors[anchor.text]
         end
       end
 
-      def update_anchor_loc(bib, eref, docid)
-        loc = eref.at(".//xmlns:locality[@type = 'anchor']") or
-          return update_anchor_create_loc(bib, eref, docid)
-        ref = loc.at("./xmlns:referenceFrom") or return
-        anchor = suffix_anchor(ref, docid)
-        a = @files.get(docid, :anchors) or return
-        a.inject([]) { |m, (_, x)| m + x.values }
-          .include?(anchor) or return
-        ref.content = anchor
+      def update_anchors_prep(docid)
+        @concat_anchors = {}
+        [@files.get(docid)&.key?(:anchors_lookup), @files.url?(docid),
+         Metanorma::Utils::to_ncname(docid)]
       end
 
-      def suffix_anchor(ref, docid)
-        @ncnames[docid] ||= Metanorma::Utils::to_ncname(docid)
-        anchor = ref.text
-        @files.url?(docid) or anchor = "#{@ncnames[docid]}_#{anchor}"
-        anchor
+      # encode both prefix and suffix to NCName
+      def suffix_anchor_indirect(prefix, suffix)
+        k = "#{prefix}_#{suffix}"
+        @ncnames[k] ||= Metanorma::Utils::to_ncname(k)
       end
 
       # if there is a crossref to another document, with no anchor, retrieve the
@@ -237,10 +258,9 @@ module Metanorma
         type = ins.at(ns("./locality/@type"))&.text
         type = "clause" if type == "annex"
         ref = ins.at(ns("./locality/referenceFrom"))&.text
-        anchor = @files.get(docid, :anchors).dig(type, ref) or return
-        ins << "<locality type='anchor'><referenceFrom>#{anchor.sub(/^_/,
-                                                                    '')}" \
-               "</referenceFrom></locality>"
+        a = @files.get(docid, :anchors).dig(type, ref) or return
+        ins << "<locality type='anchor'><referenceFrom>#{a}" \
+          "</referenceFrom></locality>"
       end
     end
   end