metanorma 2.0.0 → 2.5.5

data/lib/metanorma/collection/renderer/fileprocess.rb

@@ -3,6 +3,9 @@
 require "isodoc"
 require "metanorma-utils"
 require_relative "fileparse"
+require_relative "filelocation"
+require_relative "../artifact_store"
+require "json"
 
 module Metanorma
   class Collection
@@ -10,15 +13,78 @@ module Metanorma
       # compile and output individual file in collection
       # warn "metanorma compile -x html #{f.path}"
       def file_compile(file, filename, identifier)
-        @files.get(identifier, :sectionsplit) and return
-        opts = {
-          format: :asciidoc,
-          extension_keys: @format,
-          output_dir: @outdir,
-        }.merge(compile_options_update(identifier))
+        sectionsplit_val = @files.get(identifier, :sectionsplit)
+        sectionsplit_val and return
+        opts = compile_options_base(identifier)
+          .merge(compile_options_update(identifier))
+
+        err_count = @compile.errors.size
         @compile.compile file, opts
         @files.set(identifier, :outputs, {})
-        file_compile_formats(filename, identifier)
+        file_compile_formats(filename, identifier, opts)
+        file_compile_verify(identifier, @compile.errors[err_count..] || [])
+      end
+
+      # A member whose rendering raised produces no output file, but the
+      # collection build used to continue, exit 0, and link the absent
+      # document from index.html -- silent data loss (#586). Two
+      # complementary checks close that hole. (1) Compile pools rescued
+      # error messages in @compile.errors for its caller to surface; the
+      # standalone CLI does, but the collection pipeline never read the
+      # pool. The pool is shared across members and never reset, so the
+      # slice appended during this member's compile is this member's
+      # errors -- sound while members compile sequentially (the current
+      # design; @compile's own format-level parallelism is joined before
+      # compile returns). (2) Independently of whether anything was
+      # pooled, verify the outputs registered for the member exist on
+      # disk. Failures are reported here and accumulated; the collection
+      # renders every remaining member before Collection#render aborts,
+      # so one run reports all failed members.
+      def file_compile_verify(identifier, errors)
+        # Section-split parts (marked with :parentid) register format
+        # slots they never produce -- their :xml slot in particular is
+        # never written, and under filename renaming it aliases to the
+        # parent's name -- so verifying them is a false positive. #586
+        # was scoped to the non-sectionsplit path; parts are excluded
+        # like their parents (which return early above).
+        @files.get(identifier, :parentid) and return
+        missing = (@files.get(identifier, :outputs) || {})
+          .reject { |_fmt, path| File.exist?(path) }
+        errors.empty? && missing.empty? and return
+        file_compile_failure_log(identifier, errors, missing)
+        msg = file_compile_failure_msg(identifier, errors, missing)
+        ::Metanorma::Util.log("[metanorma] Error: #{msg}", :error)
+        fatal_errors << msg
+      end
+
+      def file_compile_failure_log(identifier, errors, missing)
+        missing.each_value do |path|
+          @log&.add("METANORMA_4", nil, params: [identifier, path])
+        end
+        errors.each do |e|
+          @log&.add("METANORMA_5", nil, params: [identifier, e])
+        end
+      end
+
+      def file_compile_failure_msg(identifier, errors, missing)
+        ret = []
+        missing.empty? or
+          ret << "expected output not generated: " \
+                 "#{missing.values.join(', ')}"
+        ret += errors
+        "Failed to render collection member #{identifier}: #{ret.join('; ')}"
+      end
+
+      def compile_options_base(identifier)
+        e = @files.get(identifier, :extract_opts)
+        {
+          format: :asciidoc,
+          extension_keys: @files.get(identifier, :format),
+          fonts: e&.dig(:fonts),
+          output_dir: @outdir,
+          pdffile: @files.get(identifier, :pdffile),
+          type: Util::taste2flavor(@flavor),
+        }
       end
 
       def compile_options_update(identifier)
@@ -33,26 +99,14 @@ module Metanorma
         ret
       end
 
-      def file_compile_formats(filename, identifier)
-        f = @files.get(identifier, :outputs)
-        @format << :presentation if @format.include?(:pdf)
-        @format.each do |e|
-          ext = @compile.processor.output_formats[e]
-          fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
-          (/html$/.match?(ext) && @files.get(identifier, :sectionsplit)) or
-            f[e] = File.join(@outdir, fn)
-        end
-        @files.set(identifier, :outputs, f)
-      end
-
-      def copy_file_to_dest(identifier)
-        out = Pathname.new(@files.get(identifier, :out_path)).cleanpath
-        out.absolute? and
-          out = out.relative_path_from(File.expand_path(FileUtils.pwd))
-        dest = File.join(@outdir, @disambig.source2dest_filename(out.to_s))
-        FileUtils.mkdir_p(File.dirname(dest))
-        FileUtils.cp @files.get(identifier, :ref), dest
-      end
+      #       def allowed_extension_keys
+      #         ret = @format.dup
+      #         @directives.detect { |d| d.key == "individual-pdf" } or
+      #           ret.delete(:pdf)
+      #         @directives.detect { |d| d.key == "individual-doc" } or
+      #           ret.delete(:doc)
+      #         ret
+      #       end
 
       # process each file in the collection
       # files are held in memory, and altered as postprocessing
@@ -67,22 +121,55 @@ module Metanorma
           if @files.get(ident, :attachment) then copy_file_to_dest(ident)
           else
             file, fname = @files.targetfile_id(ident, read: true)
+            @preserve_unresolved && member_stored?(ident, file) and next
             warn "\n\n\n\n\nProcess #{fname}: #{DateTime.now.strftime('%H:%M:%S')}"
             collection_xml = update_xrefs(file, ident, internal_refs)
-            collection_filename = File.basename(fname, File.extname(fname))
+            @preserve_unresolved and
+              store_member_artefacts(ident, file, collection_xml)
+            # Strip .xml or .html extension, but NOT section numbers like .0
+            fname_base = File.basename(fname)
+            collection_filename = fname_base
+            /\.(xml|html)$/.match?(fname_base) and
+              collection_filename = fname_base.sub(/\.(xml|html)$/, "")
             collection_xml_path = File.join(Dir.tmpdir,
                                             "#{collection_filename}.xml")
             File.write collection_xml_path, collection_xml, encoding: "UTF-8"
             file_compile(collection_xml_path, fname, ident)
-            FileUtils.rm(collection_xml_path)
+            FileUtils.rm_f(collection_xml_path)
           end
         end
       end
 
+      # Persist an isolated member's stub-bearing Semantic XML and its anchor
+      # contribution to the content-addressed store, so a later reinflation pass
+      # can relink the stubs and a re-run can resume without recompiling.
+      # Within-campaign content key for a member. (Cross-build soundness needs
+      # the full input closure -- source + includes + templates + schemas +
+      # versions; see the plan gotcha. Here the source bytes + tool version
+      # suffice because inputs are frozen across a single build campaign.)
+      def member_content_hash(source)
+        ArtifactStore.content_hash(source, ::Metanorma::VERSION)
+      end
+
+      # Resume predicate: is this member already compiled and stored for its
+      # current input content? If so the isolated compile can skip it.
+      def member_stored?(ident, source)
+        @artifact_store.key?(ident, member_content_hash(source), :semantic)
+      end
+
+      def store_member_artefacts(ident, source, semantic_xml)
+        hash = member_content_hash(source)
+        @artifact_store.write(ident, hash, :semantic, semantic_xml)
+        @artifact_store.write(ident, hash, :anchors,
+                              JSON.generate(@files.get(ident, :anchors) || {}))
+        @keep_cache or @artifact_store.prune_superseded(ident, hash)
+      end
+
       # gather internal bibitem references
       def gather_internal_refs
         @files.keys.each_with_object({}) do |i, refs|
           @files.get(i, :attachment) and next
+          @files.get(i, :sectionsplit) and next
           file, = @files.targetfile_id(i, read: true)
           gather_internal_refs1(file, i, refs)
         end
@@ -131,7 +218,7 @@ module Metanorma
           ids.each do |id, key|
             key and next
             refs[schema][id] = "Missing:#{schema}:#{id}"
-            @log&.add("Cross-References", nil, refs[schema][id])
+            @log&.add("METANORMA_1", nil, params: [refs[schema][id]])
           end
         end
         refs
@@ -151,23 +238,84 @@ module Metanorma
       def locate_internal_refs1_prep(file)
         xml = Nokogiri::XML(file, &:huge)
         r = xml.root["document_suffix"]
-        xml.xpath("//*[@id]").each_with_object({}) do |i, x|
-          /^semantic_/.match?(i.name) and next
+        ret = xml.xpath("//*[@id]").each_with_object({}) do |i, x|
           x[i["id"]] = i
           r and x[i["id"].sub(/_#{r}$/, "")] = i
         end
+        xml.xpath("//*[@anchor]").each do |i|
+          ret[i["anchor"]] = i
+          r and ret[i["anchor"].sub(/_#{r}$/, "")] = i
+        end
+        ret
       end
 
       def update_bibitem(bib, identifier)
-        docid = get_bibitem_docid(bib, identifier) or return
-        newbib = dup_bibitem(docid, bib)
-        url = @files.url(docid, relative: true,
-                                doc: !@files.get(docid, :attachment))
-        dest = newbib.at("./docidentifier") || newbib.at(ns("./docidentifier"))
+        newbib, url, attachment = update_bibitem_prep(bib, identifier)
+        newbib or return
+        dest = begin
+          newbib.at("./docidentifier") || newbib.at(ns("./docidentifier"))
+        rescue StandardError
+          nil
+        end
         dest or dest = newbib.elements[-1]
+        # An attachment carries both a <uri type="attachment"> (which drives
+        # isodoc's fmt-link attachment="true") and the citation uri; the
+        # attachment uri goes first. metanorma/iso-10303#208.
+        attachment and dest.previous = "<uri type='attachment'>#{url}</uri>"
         dest.previous = "<uri type='citation'>#{url}</uri>"
         bib.replace(newbib)
       end
+
+      def update_bibitem_prep(bib, identifier)
+        docid = get_bibitem_docid(bib, identifier) or return [nil, nil, nil]
+        newbib = dup_bibitem(docid, bib)
+        attachment = @files.get(docid, :attachment)
+        url = @files.url(docid, relative: true, doc: !attachment)
+        current_html = referencing_html_location(identifier, attachment)
+        url = make_relative_path(current_html, url) if current_html
+        [newbib, url, attachment]
+      end
+
+      # Location of the HTML that will carry the cross-reference to the target,
+      # used as the base for relativising the attachment/citation URL.
+      #
+      # Normally this is the referencing document's own output (:outputs[:html]
+      # after compilation, else :out_path converted to .html before it).
+      #
+      # But when the referencing document is sectionsplit, its content is not
+      # emitted at its own out_path: it is split into section files written at
+      # the sectionsplit output location (the collection root, unless a
+      # sectionsplit_filename directory is configured). Relativising an
+      # attachment URL against the unsplit out_path then overshoots -- the
+      # ../../ climbs above the split file and the attachment link breaks
+      # (https://github.com/metanorma/iso-10303/issues/208). So for attachment
+      # targets of a sectionsplit document, base the relative path on the
+      # sectionsplit output directory instead.
+      def referencing_html_location(identifier, attachment)
+        attachment && @files.get(identifier, :sectionsplit) and
+          return sectionsplit_ref_html(identifier)
+        document_ref_html(identifier)
+      end
+
+      # The split section files sit at the sectionsplit output directory: the
+      # collection root, or the directory of a configured sectionsplit_filename.
+      def sectionsplit_ref_html(identifier)
+        d = @files.preserve_directory_structure?(identifier)
+        d ? File.join(File.dirname(d), "x.html") : "x.html"
+      end
+
+      # Use :outputs[:html] if available (after compilation), otherwise convert
+      # :out_path to HTML (before compilation), following ref_file_xml2html.
+      def document_ref_html(identifier)
+        @files.get(identifier, :outputs)&.dig(:html) ||
+          out_path_to_html(@files.get(identifier, :out_path))
+      end
+
+      def out_path_to_html(out_path)
+        out_path or return nil
+        out_path.end_with?(".xml") and return out_path.sub(/\.xml$/, ".html")
+        "#{out_path}.html"
+      end
     end
   end
 end

data/lib/metanorma/collection/renderer/navigation.rb

@@ -4,10 +4,10 @@ module Metanorma
       # @param elm [Nokogiri::XML::Element]
       # @return [String]
       def indexfile_title(entry)
-        if entry.bibdata &&
-            x = entry.bibdata.title.detect { |t| t.type == "main" } ||
-                entry.bibdata.title.first
-          x.title.content
+        titles = entry.bibdata && Array(entry.bibdata.title)
+        if titles && !titles.empty? &&
+            (x = titles.detect { |t| t.type == "main" } || titles.first)
+          x.content
         else
           entry.title
         end
@@ -39,9 +39,23 @@ module Metanorma
         @c.decode(@isodoc.docid_prefix(nil, ident))
       end
 
+      # Check if file has a recognized MIME type (other than XML)
+      # If so, don't append .html (e.g., .svg, .png, .jpg, etc.)
       def index_link(docref, ident)
         if docref.file
-          @files.get(ident, :out_path).sub(/\.xml$/, ".html")
+          out_path = @files.get(ident, :out_path)
+          # Ensure the path ends with .html for documents, but not for recognized file types
+          if Util::mime_file_recognised?(out_path) && !out_path.end_with?(".xml")
+            # File has a recognized extension (like .svg, .png), keep it as is
+            out_path
+          elsif out_path.end_with?(".xml")
+            out_path.sub(/\.xml$/, ".html")
+          elsif out_path.end_with?(".html")
+            out_path
+          else
+            "#{out_path}.html"
+          end
+
         else "#{docref.id}.html"
         end
       end
@@ -55,7 +69,7 @@ module Metanorma
 
       def index?(mnf)
         mnf.index and return true
-        mnf.entry.detect { |e| index?(e) }
+        mnf.entry&.detect { |e| index?(e) }
       end
 
       def indexfile1(mnf)
@@ -114,7 +128,7 @@ module Metanorma
             files.each { |f| docrefs(f, u) }
           end
         end
-        r.doc.root&.to_html&.gsub("\n", " ")
+        r.doc.root&.to_html&.tr("\n", " ")
       end
 
       def liquid_docrefs(mnfs)

data/lib/metanorma/collection/renderer/render_word.rb

@@ -1,11 +1,13 @@
 module Metanorma
   class Collection
     class Renderer
-      def docconv
+      def docconv(added_options = {})
         @tempfile_cache ||= []
-        doctype = @doctype.to_sym
-        x = Asciidoctor.load nil, backend: doctype
-        x.converter.doc_converter(DocOptionsNode.new(@directives, @dirname))
+        flavor = Util::taste2flavor(@flavor).to_sym
+        opts = Util::taste2isodoc_attrs(@flavor, :doc)
+        x = Asciidoctor.load nil, backend: flavor
+        x.converter.doc_converter(DocOptionsNode.new(@directives, @dirname,
+                                                     added_options.merge(opts)))
       end
 
       def concat_extract_files(filename)
@@ -34,7 +36,8 @@ module Metanorma
 
       def wrapping_doc_body(doc)
         doc.xpath(ns("//annex | //preface | //bibliography")).each(&:remove)
-        s = doc.at(ns("//sections"))
+        s = doc.at(ns("//sections")) ||
+          doc.root.elements[-1].after("<sections/>").next
         repl = <<~BODY
           <sections><clause id='_collection_placeholder'><p>PLACEHOLDER</p></clause></sections>
         BODY
@@ -50,8 +53,8 @@ module Metanorma
         doc
       end
 
-      SECTION_BREAK = '<p class="MsoNormal"><br clear="all" class="section"/></p>'
-        .freeze
+      SECTION_BREAK =
+        '<p class="MsoNormal"><br clear="all" class="section"/></p>'.freeze
       DIV1 = '<div class="WordSection1">&#xa0;</div>'.freeze
       DIV2 = '<div class="WordSection2">&#xa0;</div>'.freeze
 
@@ -69,10 +72,10 @@ module Metanorma
       end
 
       def collection_coverpages(conv, docs)
-        conv.wordintropage and [DIV2, SECTION_BREAK].reverse.each do |s|
+        conv.wordintropage and [DIV2, SECTION_BREAK].reverse_each do |s|
           docs.unshift(Nokogiri::XML(s).root)
         end
-        conv.wordcoverpage and [DIV1, SECTION_BREAK].reverse.each do |s|
+        conv.wordcoverpage and [DIV1, SECTION_BREAK].reverse_each do |s|
           docs.unshift(Nokogiri::XML(s).root)
         end
         docs
@@ -111,7 +114,7 @@ module Metanorma
       end
 
       class DocOptionsNode
-        def initialize(directives, dir)
+        def initialize(directives, dir, options)
           @dir = dir
           @wordcoverpage =
             Util::hash_key_detect(directives, "document-word-coverpage",
@@ -119,12 +122,14 @@ module Metanorma
           @wordintropage =
             Util::hash_key_detect(directives, "document-word-intropage",
                                   @wordintropage)
+          @options = options
         end
 
         def attr(key)
           case key
           when "wordcoverpage" then Util::rel_path_resolve(@dir, @wordcoverpage)
           when "wordintropage" then Util::rel_path_resolve(@dir, @wordintropage)
+          else @options[key.to_sym]
           end
         end
       end