metanorma 2.0.0 → 2.5.5

data/lib/metanorma/collection/filelookup/filelookup.rb

@@ -1,7 +1,9 @@
 require "isodoc"
 require "htmlentities"
 require "metanorma-utils"
+require "marcel"
 require_relative "filelookup_sectionsplit"
+require_relative "utils"
 
 module Metanorma
   class Collection
@@ -18,38 +20,54 @@ module Metanorma
         @parent = parent
         @xml = parent.xml
         @isodoc = parent.isodoc
+        @isodoc_presxml = parent.isodoc_presxml
         @path = path
         @compile = parent.compile
         @documents = parent.documents
         @files_to_delete = []
         @disambig = Util::DisambigFiles.new
         @manifest = parent.manifest
-        read_files(@manifest.entry)
+        read_files(@manifest.entry, parent.manifest)
       end
 
-      def read_files(entries)
+      def read_files(entries, parent, idx = 0)
         Array(entries).each do |e|
-          e.file and read_file(e)
-          read_files(e.entry)
+          derive_format(e, parent)
+          if e.file
+            read_file(e, idx)
+            idx += 1
+          end
+          idx = read_files(e.entry, e, idx)
         end
+        idx
       end
 
-      def read_file(manifest)
+      def derive_format(entry, parent)
+        entry.attachment and return
+        if Array(entry.format).empty?
+          parent_fmt = Array(parent.format)
+          entry.format = parent_fmt.empty? ? %w(xml presentation html) : parent_fmt.dup
+        end
+        entry.format |= ["xml", "presentation"]
+      end
+
+      def read_file(manifest, idx)
         i, k = read_file_idents(manifest)
-        entry = file_entry(manifest, k) or return
+        entry = file_entry(manifest, k, idx) or return
         bibdata_process(entry, i)
         bibitem_process(entry)
-        @files[key(i)] = entry
+        @files[entry_key(i)] = entry
       end
 
       def read_file_idents(manifest)
         id = manifest.identifier
-        sanitised_id = key(@isodoc.docid_prefix("", manifest.identifier.dup))
+        sanitised_id =
+          entry_key(@isodoc.docid_prefix("", manifest.identifier.dup))
         #       if manifest.bibdata and # NO, DO NOT FISH FOR THE GENUINE IDENTIFIER IN BIBDATA
         #         d = manifest.bibdata.docidentifier.detect { |x| x.primary } ||
         #           manifest.bibdata.docidentifier.first
         #         k = d.id
-        #         i = key(@isodoc.docid_prefix(d.type, d.id.dup))
+        #         i = entry_key(@isodoc.docid_prefix(d.type, d.id.dup))
         #       end
         [id, sanitised_id]
       end
@@ -62,12 +80,18 @@ module Metanorma
           file, _filename = targetfile(entry, read: true)
           xml = Nokogiri::XML(file, &:huge)
           add_document_suffix(ident, xml)
-          entry.merge!(anchors: read_anchors(xml), ids: read_ids(xml),
-                       bibdata: xml.at(ns("//bibdata")),
-                       document_suffix: xml.root["document_suffix"])
+          entry.merge!(bibdata_extract(xml))
         end
       end
 
+      def bibdata_extract(xml)
+        anchors = read_anchors(xml)
+        { anchors: anchors, anchors_lookup: anchors_lookup(anchors),
+          ids: read_ids(xml),
+          bibdata: xml.at(ns("//bibdata")),
+          document_suffix: xml.root["document_suffix"] }
+      end
+
       def bibitem_process(entry)
         entry[:bibitem] = entry[:bibdata].dup
         entry[:bibitem].name = "bibitem"
@@ -76,57 +100,138 @@ module Metanorma
       end
 
       # ref is the absolute source file address
-      # rel_path is the relative source file address, relative to the YAML locaton
+      # rel_path is the relative source file address, relative to the YAML location
       # out_path is the destination file address, with any references outside
       # the working directory (../../...) truncated, and based on relative path
       # identifier is the id with only spaces, no nbsp
-      def file_entry(ref, identifier)
+      # idx is the index of the document in the manifest
+      def file_entry(ref, identifier, idx)
         ref.file or return
         abs = @documents[Util::key identifier].file
+        # For sectionsplit outputs from YAML manifest, we need to compute the full path
+        # by combining sectionsplit_filename directory with ref.file basename
+        sso = ref.respond_to?(:sectionsplit_output) && ref.sectionsplit_output
+        out_path, rel_path = file_entry_paths(ref, idx, sso)
         ret = if ref.file
-                { type: "fileref", ref: abs, rel_path: ref.file, url: ref.url,
-                  out_path: output_file_path(ref) }
+                { type: "fileref", ref: abs, rel_path: rel_path, url: ref.url,
+                  out_path: out_path, idx: idx,
+                  output_filename: ref.output_filename,
+                  sectionsplit_filename: ref.sectionsplit_filename,
+                  pdffile: ref.pdffile, format: ref.format&.map(&:to_sym) }
+                  .compact
               else { type: "id", ref: ref.id }
               end
         file_entry_copy(ref, ret)
         ret.compact
       end
 
+
+      # ref is the absolute source file address
+      # rel_path is the relative source file address, relative to the YAML location
+      # out_path is the destination file address, with any references outside
+      # the working directory (../../...) truncated, and based on relative path
+      # identifier is the id with only spaces, no nbsp
+      # extract_opts are the compilation options extracted as document attributes
+      def file_entry_struct(ref, abs)
+        adoc = abs.sub(/\.xml$/, ".adoc")
+        if adoc.end_with?(".adoc") && File.exist?(adoc)
+          opts = Metanorma::Input::Asciidoc.new.extract_options(File.read(adoc))
+        end
+        { type: "fileref", ref: abs, rel_path: ref.file, url: ref.url,
+          out_path: output_file_path(ref), pdffile: ref.pdffile,
+          format: ref.format&.map(&:to_sym), extract_opts: opts }.compact
+      end
+
+      def file_entry_paths(ref, idx, sso)
+        base = File.basename(ref.file, ".xml")
+        if sso && ref.respond_to?(:sectionsplit_filename) &&
+            ref.sectionsplit_filename
+          # Extract directory from sectionsplit_filename
+          dir = File.dirname(ref.sectionsplit_filename)
+          if dir == "." # No directory in pattern
+            [output_file_path(ref, idx), ref.file]
+          else # Pattern has directory, prepend it
+            full_path = File.join(dir, base)
+            [full_path, "#{full_path}.xml"]
+          end
+        else [output_file_path(ref, idx), ref.file]
+        end
+      end
+
+      # Substitute special strings in filename patterns.
+      # See Util.substitute_filename_pattern.
+      def substitute_filename_pattern(pattern, options = {})
+        Util.substitute_filename_pattern(pattern, options)
+      end
+
       # TODO make the output file location reflect source location universally,
       # not just for attachments: no File.basename
-      def output_file_path(ref)
-        f = File.basename(ref.file)
-        ref.attachment and f = ref.file
-        @disambig.source2dest_filename(f)
+      #
+      # For files with custom directory structure, construct path with directory
+      # For files with output_filename, use that (with substitutions)
+      # For others, use basename of ref.file
+      def output_file_path(ref, idx)
+        has_custom_dir, file_has_dir, params = output_file_path_prep(ref, idx)
+        # Apply sectionsplit_filename directory structure if:
+        # 1. File has sectionsplit enabled (parent document being split), OR
+        # 2. File is a sectionsplit output (from collection or single-file sectionsplit)
+        # Regular files that inherit sectionsplit_filename from collection level
+        # but are not sectionsplit outputs should NOT use it
+        is_sectionsplit_output = ref.respond_to?(:sectionsplit_output) && ref.sectionsplit_output
+        use_sectionsplit_dir = ref.sectionsplit_filename && has_custom_dir &&
+          (ref.sectionsplit || is_sectionsplit_output || file_has_dir)
+        f = if use_sectionsplit_dir
+              # For sectionsplit outputs, return just the basename
+              # The directory will be applied during file_compile_format
+              # via preserve_directory_structure?
+              File.basename(ref.file)
+            elsif ref.output_filename
+              substitute_filename_pattern(ref.output_filename, **params)
+            elsif file_has_dir
+              ref.file # Preserve directory structure already in ref.file
+            elsif ref.attachment
+              ref.file
+            else File.basename(ref.file)
+            end
+        ret = @disambig.source2dest_filename(f, preserve_dirs: ref.attachment)
+        warn ret
+        ret
+      end
+
+      def output_file_path_prep(ref, idx)
+        b = File.basename(ref.file)
+        b_no_ext = File.basename(ref.file, ".*")
+        # Check for sectionsplit_filename (for both parent and split output files)
+        # or output_filename
+        custom_filename = ref.sectionsplit_filename || ref.output_filename
+        has_custom_dir = custom_filename && File.dirname(custom_filename) != "."
+        # Also check if ref.file itself contains a directory
+        file_has_dir = File.dirname(ref.file) != "."
+        params = { document_num: idx, basename: b_no_ext, basename_legacy: b }
+        [has_custom_dir, file_has_dir, params]
       end
 
       def file_entry_copy(ref, ret)
         %w(attachment sectionsplit index presentation-xml url
-           bare-after-first).each do |s|
+           bare-after-first output_filename sectionsplit_filename
+           sectionsplit_output).each do |s|
           ref.respond_to?(s.to_sym) and
-            ret[s.gsub("-", "").to_sym] = ref.send(s)
+            ret[s.delete("-").to_sym] = ref.send(s)
         end
       end
 
       def add_document_suffix(identifier, doc)
         document_suffix = Metanorma::Utils::to_ncname(identifier)
-        Metanorma::Utils::anchor_attributes.each do |(tag_name, attribute_name)|
-          Util::add_suffix_to_attributes(doc, document_suffix, tag_name,
-                                         attribute_name, @isodoc)
+        ids = doc.xpath("./@id | .//@id").map(&:value)
+        Util::anchor_id_attributes.each do |(tag_name, attr_name)|
+          Util::add_suffix_to_attrs(doc, document_suffix, tag_name, attr_name,
+                                    @isodoc)
         end
-        url_in_css_styles(doc, document_suffix)
+        Util::url_in_css_styles(doc, ids, document_suffix)
         doc.root["document_suffix"] ||= ""
         doc.root["document_suffix"] += document_suffix
       end
 
-      # update relative URLs, url(#...), in CSS in @style attrs (including SVG)
-      def url_in_css_styles(doc, document_suffix)
-        doc.xpath("//*[@style]").each do |s|
-          s["style"] = s["style"]
-            .gsub(%r{url\(#([^)]+)\)}, "url(#\\1_#{document_suffix})")
-        end
-      end
-
       # return citation url for file
       # @param doc [Boolean] I am a Metanorma document,
       # so my URL should end with html or pdf or whatever
@@ -135,13 +240,6 @@ module Metanorma
         data[:url] || targetfile(data, options)[1]
       end
 
-      # are references to the file to be linked to a file in the collection,
-      # or externally? Determines whether file suffix anchors are to be used
-      def url?(ident)
-        data = get(ident) or return false
-        data[:url]
-      end
-
       # return file contents + output filename for each file in the collection,
       # given a docref entry
       # @param data [Hash] docref entry
@@ -155,7 +253,7 @@ module Metanorma
         options = { read: false, doc: true, relative: false }.merge(options)
         path = options[:relative] ? data[:rel_path] : data[:ref]
         if data[:type] == "fileref"
-          ref_file path, data[:out_path], options[:read], options[:doc]
+          ref_file path, data, options[:read], options[:doc]
         else
           xml_file data[:id], options[:read]
         end
@@ -165,13 +263,40 @@ module Metanorma
         targetfile(get(ident), options)
       end
 
-      def ref_file(ref, out, read, doc)
+      def ref_file(ref, data, read, doc)
         file = File.read(ref, encoding: "utf-8") if read
-        filename = out.dup
-        filename.sub!(/\.xml$/, ".html") if doc
+        # Use the actual output path from :outputs if available (set after compilation)
+        # Otherwise fall back to :out_path (set at initialization)
+        filename = if doc && data[:outputs] && data[:outputs][:html]
+                     data[:outputs][:html].sub(
+                       %r{^#{Regexp.escape(@parent.outdir)}/}, ""
+                     )
+                   else
+                     data[:out_path].dup
+                   end
+        if doc && !data[:outputs]
+          filename = ref_file_xml2html(filename)
+        end
         [file, filename]
       end
 
+      # Check if file has a recognized MIME type (other than XML)
+      # If so, don't append .html (e.g., .svg, .png, .jpg, etc.)
+      # Only process if it doesn't have a recognized non-XML extension
+      # If filename ends in .xml, replace with .html
+      # Otherwise (including sectionsplit files like "file.xml.0" or
+      # custom titles), append .html
+      def ref_file_xml2html(filename)
+        unless Util::mime_file_recognised?(filename) &&
+            !filename.end_with?(".xml")
+          filename = if filename.end_with?(".xml")
+                       filename.sub(/\.xml$/, ".html")
+                     else "#{filename}.html"
+                     end
+        end
+        filename
+      end
+
       def xml_file(id, read)
         file = @xml.at(ns("//doc-container[@id = '#{id}']")).to_xml if read
         filename = "#{id}.html"
@@ -198,10 +323,10 @@ module Metanorma
         ret[val[:type]] ||= {}
         index = if val[:container] || val[:label].nil? || val[:label].empty?
                   UUIDTools::UUID.random_create.to_s
-                else val[:label]
+                else val[:label].gsub(%r{<[^>]+>}, "")
                 end
         ret[val[:type]][index] = key
-        ret[val[:type]][val[:value]] = key if val[:value]
+        v = val[:value] and ret[val[:type]][v.gsub(%r{<[^>]+>}, "")] = key
       end
 
       # Also parse all ids in doc (including ones which won't be xref targets)
@@ -209,41 +334,24 @@ module Metanorma
         ret = {}
         xml.traverse do |x|
           x.text? and next
-          /^semantic__/.match?(x.name) and next
           x["id"] and ret[x["id"]] = true
         end
         ret
       end
 
-      def key(ident)
-        @c.decode(ident).gsub(/(\p{Zs})+/, " ").sub(/^metanorma-collection /,
-                                                    "")
-      end
-
-      def keys
-        @files.keys
-      end
-
-      def get(ident, attr = nil)
-        if attr then @files[key(ident)][attr]
-        else @files[key(ident)]
-        end
-      end
-
-      def set(ident, attr, value)
-        @files[key(ident)][attr] = value
-      end
-
-      def each
-        @files.each
-      end
-
-      def each_with_index
-        @files.each_with_index
-      end
-
-      def ns(xpath)
-        @isodoc.ns(xpath)
+      # Check if we should preserve directory structure for an identifier
+      # Returns the custom filename if directory structure should be preserved,
+      # nil otherwise
+      def preserve_directory_structure?(ident)
+        ret = if get(ident, :sectionsplit_output)
+                # For sectionsplit outputs, use rel_path which has the directory
+                get(ident, :rel_path) || get(ident, :out_path)
+              elsif get(ident, :sectionsplit)
+                get(ident, :sectionsplit_filename)
+              else get(ident, :output_filename)
+              end
+        # Return the custom filename only if it contains a directory
+        ret && File.dirname(ret) != "." ? ret : nil
       end
     end
   end

data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb

@@ -6,8 +6,8 @@ module Metanorma
       def add_section_split
         ret = @files.keys.each_with_object({}) do |k, m|
           if @files[k][:sectionsplit] && !@files[k][:attachment]
-            process_section_split_instance(k, m)
-            cleanup_section_split_instance(k, m)
+            original_out_path = process_section_split_instance(k, m)
+            cleanup_section_split_instance(k, m, original_out_path)
           end
           m[k] = @files[k]
         end
@@ -15,67 +15,139 @@ module Metanorma
       end
 
       def process_section_split_instance(key, manifest)
+        # Save the original out_path before it gets modified
+        original_out_path = @files[key][:out_path]
         s, sectionsplit_manifest = sectionsplit(key)
         s.each_with_index do |f1, i|
           add_section_split_instance(f1, manifest, key, i)
         end
-        manifest["#{key}:index.html"] =
-          add_section_split_cover(sectionsplit_manifest, key)
+        a = add_section_split_attachments(sectionsplit_manifest, key) and
+          manifest["#{key}:attachments"] = a
+        add_section_split_cover(manifest, sectionsplit_manifest, key)
+        # Return the original path for cleanup
+        original_out_path
       end
 
-      def cleanup_section_split_instance(key, manifest)
+      def cleanup_section_split_instance(key, manifest, original_out_path)
+        # Delete the sectionsplit index.html from source directory after it's copied to output
         @files_to_delete << manifest["#{key}:index.html"][:ref]
+        # Delete the original files when sectionsplit happens (all formats: html, xml, presentation.xml)
+        # Use the saved original out_path (before it was changed to index.html)
+        if original_out_path
+          base = File.join(@parent.outdir, original_out_path.sub(/\.xml$/, ""))
+          @files_to_delete << "#{base}.html"
+          @files_to_delete << "#{base}.xml"
+          @files_to_delete << "#{base}.presentation.xml"
+        end
         # @files[key].delete(:ids).delete(:anchors)
         @files[key][:indirect_key] = @sectionsplit.key
+        # sectionsplit is HTML-only; hand the parent its whole Presentation XML
+        # so the collection PDF/presentation concatenation inlines the document
+        # intact (raw_file) instead of degrading to a bibdata-only, namespace-
+        # less stub. file_compile early-returns for sectionsplit, so this
+        # :outputs entry is what concatenate1 consumes.
+        # https://github.com/metanorma/iso-10303/issues/208
+        @files[key][:outputs] =
+          { presentation: @sectionsplit.whole_presentation_file }
       end
 
-      def add_section_split_cover(manifest, ident)
+      def add_section_split_cover(manifest, sectionsplit_manifest, ident)
         cover = @sectionsplit
-          .section_split_cover(manifest, @parent.dir_name_cleanse(ident),
+          .section_split_cover(sectionsplit_manifest,
+                               @parent.dir_name_cleanse(ident),
                                one_doc_collection?)
         @files[ident][:out_path] = cover
-        { attachment: true, index: false, out_path: cover,
-          ref: File.join(File.dirname(manifest.file), cover) }
+        src = File.join(File.dirname(sectionsplit_manifest.file), cover)
+        m = { attachment: true, index: false, out_path: cover, ref: src }
+        manifest["#{ident}:index.html"] = m
+        one_doc_collection? and
+          add_cover_one_doc_coll(manifest, sectionsplit_manifest, ident, m)
+      end
+
+      def add_cover_one_doc_coll(manifest, sectionsplit_manifest, key, entry)
+        idx = File.join(File.dirname(sectionsplit_manifest.file), "index.html")
+        FileUtils.cp entry[:ref], idx
+        manifest["#{key}:index1.html"] =
+          entry.merge(out_path: "index.html", ref: idx)
       end
 
       def one_doc_collection?
-        return false
         docs = 0
         @files.each_value do |v|
           v[:attachment] and next
           v[:presentationxml] and next
           docs += 1
         end
-        docs > 1
+        docs <= 1
+      end
+
+      def add_section_split_attachments(manifest, ident)
+        attachments = @sectionsplit
+          .section_split_attachments(out: File.dirname(manifest.file))
+        attachments or return
+        @files[ident][:out_path] = attachments
+        { attachment: true, index: false, out_path: attachments,
+          ref: File.join(File.dirname(manifest.file), attachments) }
       end
 
       def add_section_split_instance(file, manifest, key, idx)
-        presfile, newkey, xml =
-          add_section_split_instance_prep(file, key)
-        manifest[newkey] =
-          { parentid: key, presentationxml: true, type: "fileref",
-            rel_path: file[:url], out_path: File.basename(file[:url]),
-            anchors: read_anchors(xml), ids: read_ids(xml),
-            sectionsplit_output: true,
-            bibdata: @files[key][:bibdata], ref: presfile }
-        @files_to_delete << file[:url]
-        manifest[newkey][:bare] = true unless idx.zero?
+        presfile, newkey, xml = add_section_split_instance_prep(file, key)
+        anchors = read_anchors(xml)
+        # Preserve directory structure in out_path if parent has custom sectionsplit_filename with directory
+        sectionsplit_fname = @files[key][:sectionsplit_filename]
+
+        # file[:url] from sectionsplit.rb already has placeholders substituted and includes full path
+        # Use it directly for out_path (without .xml extension)
+        base_filename = File.basename(file[:url], ".xml")
+
+        # Get the directory from file[:url] which already has placeholders substituted
+        file_dir = File.dirname(file[:url])
+
+        # If file[:url] has a directory (i.e., placeholders were substituted), use it
+        out_path_value = if file_dir == "."
+                           base_filename
+                         else
+                           File.join(file_dir, base_filename)
+                         end
+
+        m = { parentid: key, presentationxml: true, type: "fileref",
+              rel_path: out_path_value, out_path: out_path_value,
+              anchors: anchors, anchors_lookup: anchors_lookup(anchors),
+              ids: read_ids(xml), format: @files[key][:format],
+              sectionsplit_output: true, indirect_key: @sectionsplit.key,
+              bibdata: @files[key][:bibdata], ref: presfile,
+              sectionsplit_filename: sectionsplit_fname,
+              idx: @files[key][:idx] }
+        m[:bare] = true unless idx.zero?
+        manifest[newkey] = m
+        # Don't delete split output files - we want to keep them!
+        # The original parent HTML file is deleted in cleanup_section_split_instance
       end
 
       def add_section_split_instance_prep(file, key)
-        presfile = File.join(File.dirname(@files[key][:ref]),
-                             File.basename(file[:url]))
-        newkey = key("#{key.strip} #{file[:title]}")
+        # XML files are always stored flat in the _files directory (no subdirectories)
+        # file[:url] contains full path with directory for HTML output, but XML is basename only
+        xml_basename = File.basename(file[:url])
+        presfile = File.join(File.dirname(@files[key][:ref]), xml_basename)
+        newkey = entry_key("#{key.strip} #{file[:title]}")
         xml = Nokogiri::XML(File.read(presfile), &:huge)
         [presfile, newkey, xml]
       end
 
       def sectionsplit(ident)
         file = @files[ident][:ref]
+        # @base must always be just basename, never contain directory components
+        # Directory structure comes from sectionsplit_filename pattern only
+        base = File.basename(@files[ident][:out_path] || file)
         @sectionsplit = ::Metanorma::Collection::Sectionsplit
-          .new(input: file, base: @files[ident][:out_path], dir: File.dirname(file),
-               output: @files[ident][:out_path], compile_opts: @parent.compile_options,
-               fileslookup: self, ident: ident, isodoc: @isodoc)
+          .new(input: file, base: base,
+               dir: File.dirname(file), output: @files[ident][:out_path],
+               compile_opts: @parent.compile_options, ident: ident,
+               fileslookup: self, isodoc: @isodoc,
+               parent_idx: @files[ident][:idx],
+               sectionsplit_filename: @files[ident][:sectionsplit_filename],
+               isodoc_presxml: @isodoc_presxml,
+               document_suffix: @files[ident][:document_suffix])
         coll = @sectionsplit.sectionsplit.sort_by { |f| f[:order] }
         xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"), &:huge)
         [coll, @sectionsplit

data/lib/metanorma/collection/filelookup/utils.rb

@@ -0,0 +1,52 @@
+module Metanorma
+  class Collection
+    class FileLookup
+      def anchors_lookup(anchors)
+        anchors.values.each_with_object({}) do |v, m|
+          v.each_value { |v1| m[v1] = true }
+        end
+      end
+
+      # are references to the file to be linked to a file in the collection,
+      # or externally? Determines whether file suffix anchors are to be used
+      def url?(ident)
+        data = get(ident) or return false
+        data[:url]
+      end
+
+      # Normalise an identifier to its @files hash key: decode entities, squeeze
+      # whitespace, and strip a leading "metanorma-collection " prefix. Distinct
+      # from Util::key, which does NOT strip that prefix.
+      def entry_key(ident)
+        @c.decode(ident).gsub(/(\p{Zs})+/, " ")
+          .sub(/^metanorma-collection /, "")
+      end
+
+      def keys
+        @files.keys
+      end
+
+      def get(ident, attr = nil)
+        if attr then @files[entry_key(ident)][attr]
+        else @files[entry_key(ident)]
+        end
+      end
+
+      def set(ident, attr, value)
+        @files[entry_key(ident)][attr] = value
+      end
+
+      def each
+        @files.each
+      end
+
+      def each_with_index
+        @files.each_with_index
+      end
+
+      def ns(xpath)
+        @isodoc.ns(xpath)
+      end
+    end
+  end
+end