metanorma 1.3.1 → 1.3.6

data/lib/metanorma/collection_fileprocess.rb

@@ -15,27 +15,71 @@ module Metanorma
     def read_files(path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       files = {}
       @xml.xpath(ns("//docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
-        files[identifier] = file_entry(d, path)
-        next if files[identifier][:attachment]
+        identifier = d.at(ns("./identifier")).children.to_xml
+        files[identifier] = file_entry(d, identifier, path)
+        if files[identifier][:attachment]
+          files[identifier][:bibdata] = Metanorma::Document
+            .attachment_bibitem(identifier).root
+        else
+          file, _filename = targetfile(files[identifier], read: true)
+          xml = Nokogiri::XML(file)
+          add_document_suffix(identifier, xml)
+          files[identifier][:anchors] = read_anchors(xml)
+          files[identifier][:bibdata] = xml.at(ns("//bibdata"))
+        end
+        files[identifier][:bibitem] = files[identifier][:bibdata].dup
+          files[identifier][:bibitem].name = "bibitem"
+          files[identifier][:bibitem]["hidden"] = "true"
+          files[identifier][:bibitem]&.at("./*[local-name() = 'ext']")&.remove
+      end
+      add_section_split(files)
+    end
 
-        file, _filename = targetfile(files[identifier], true)
-        xml = Nokogiri::XML(file)
-        add_document_suffix(identifier, xml)
-        files[identifier][:anchors] = read_anchors(xml)
-        files[identifier][:bibdata] = xml.at(ns("//bibdata"))
+    def add_section_split(files)
+      files.keys.each_with_object({}) do |k, m|
+        if files[k][:sectionsplit] == "true" && !files[k]["attachment"]
+          sectionsplit(files[k][:rel_path]).each_with_index do |f1, i|
+            m[k + f1[:title]] =
+              { parentid: k, presentationxml: true, type: "fileref",
+                rel_path: f1[:url], out_path: File.basename(f1[:url]),
+                anchors: read_anchors(Nokogiri::XML(File.read(f1[:url]))),
+                bibdata: files[k][:bibdata], ref: f1[:url] }
+            m[k + f1[:title]][:bare] = true unless i.zero?
+          end
+          m[k] = files[k]
+        else
+          m[k] = files[k]
+        end
       end
-      files
     end
 
-    def file_entry(docref, path)
-      ret = if docref["fileref"]
-              { type: "fileref", ref: File.join(path, docref["fileref"]),
-                rel_path: docref["fileref"] }
+    def sectionsplit(file)
+      Compile.new.compile(
+        file, { format: :asciidoc, extension_keys: [:presentation] }
+        .merge(@compile_options)
+      )
+      r = file.sub(/\.xml$/, ".presentation.xml")
+      @isodoc.sectionsplit(
+        Nokogiri::XML(File.read(r)), File.basename(r), File.dirname(r)
+      ).sort_by { |f| f[:order] }
+    end
+
+    # rel_path is the source file address, determined relative to the YAML.
+    # out_path is the destination file address, with any references outside
+    # the working directory (../../...) truncated
+    def file_entry(ref, identifier, _path)
+      out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
+      ret = if ref["fileref"]
+              { type: "fileref", ref: @documents[identifier].file,
+                rel_path: ref["fileref"],
+                out_path: out }
             else
-              { type: "id", ref: docref["id"] }
+              { type: "id", ref: ref["id"] }
             end
-      ret[:attachment] = docref["attachment"] if docref["attachment"]
+      ret[:attachment] = ref["attachment"] if ref["attachment"]
+      ret[:sectionsplit] = ref["sectionsplit"] if ref["sectionsplit"]
+      ret[:presentationxml] = ref["presentation-xml"] if ref["presentation-xml"]
+      ret[:bareafterfirst] = ref["bare-after-first"] if ref["bare-after-first"]
       ret
     end
 
@@ -58,12 +102,20 @@ module Metanorma
 
     # return file contents + output filename for each file in the collection,
     # given a docref entry
-    # @param data [Hash]
-    # @param read [Boolean]
+    # @param data [Hash] docref entry
+    # @param read [Boolean] read the file in and return it
+    # @param doc [Boolean] I am a Metanorma document,
+    # so my URL should end with html or pdf or whatever
+    # @param relative [Boolean] Return output path,
+    # formed relative to YAML file, not input path, relative to calling function
     # @return [Array<String, nil>]
-    def targetfile(data, read = false, doc = true)
-      if data[:type] == "fileref" then ref_file data[:ref], read, doc
-      else xml_file data[:id], read
+    def targetfile(data, options)
+      options = { read: false, doc: true, relative: false }.merge(options)
+      path = options[:relative] ? data[:rel_path] : data[:ref]
+      if data[:type] == "fileref"
+        ref_file path, data[:out_path], options[:read], options[:doc]
+      else
+        xml_file data[:id], options[:read]
       end
     end
 
@@ -71,43 +123,74 @@ module Metanorma
     # @param read [Boolean]
     # @param doc [Boolean]
     # @return [Array<String, nil>]
-    def ref_file(ref, read, doc)
+    def ref_file(ref, out, read, doc)
       file = File.read(ref, encoding: "utf-8") if read
-      filename = ref.dup
+      filename = out.dup
       filename.sub!(/\.xml$/, ".html") if doc
       [file, filename]
     end
 
     # compile and output individual file in collection
-    def file_compile(f, filename, identifier)
-      # warn "metanorma compile -x html #{f.path}"
+    # warn "metanorma compile -x html #{f.path}"
+    def file_compile(file, filename, identifier)
       c = Compile.new
-      c.compile f.path, { format: :asciidoc,
-                          extension_keys: @format }.merge(@compile_options)
+      c.compile file.path, { format: :asciidoc, extension_keys: @format }
+        .merge(compile_options(identifier))
       @files[identifier][:outputs] = {}
+      file_compile_formats(file, filename, identifier, c)
+    end
+
+    def compile_options(identifier)
+      ret = @compile_options.dup
+      Array(@directives).include?("presentation-xml") ||
+        @files[identifier][:presentationxml] and
+        ret.merge!(passthrough_presentation_xml: true)
+      @files[identifier][:sectionsplit] == "true" and
+        ret.merge!(sectionsplit: "true")
+      @files[identifier][:bare] == true and
+        ret.merge!(bare: true)
+      ret
+    end
+
+    def file_compile_formats(file, filename, identifier, compile)
       @format.each do |e|
-        ext = c.processor.output_formats[e]
-        fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
-        FileUtils.mv f.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
-        @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        ext = compile.processor.output_formats[e]
+        fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
+        if /html$/.match?(ext) && @files[identifier][:sectionsplit]
+          # file_sectionsplit_copy(file, fn, identifier, ext, e)
+        else
+          FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"),
+            File.join(@outdir, fn)
+            @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        end
       end
     end
 
+    def file_sectionsplit_copy(file, base, identifier, ext, format)
+      dir = file.path.sub(/\.xml$/, ".#{ext}_collection")
+      files = Dir.glob("#{dir}/*.#{ext}")
+      FileUtils.cp files, @outdir
+      cover = File.join(@outdir, base.sub(/\.html$/, ".index.html"))
+      FileUtils.cp File.join(dir, "index.html"), cover
+      @files[identifier][:outputs][format] = cover
+    end
+
     def copy_file_to_dest(fileref)
-      _file, filename = targetfile(fileref, true, false)
-      dest = File.join(@outdir, fileref[:rel_path])
+      dest = File.join(@outdir, fileref[:out_path])
       FileUtils.mkdir_p(File.dirname(dest))
-      FileUtils.cp filename, dest
+      FileUtils.cp fileref[:ref], dest
     end
 
     # process each file in the collection
     # files are held in memory, and altered as postprocessing
     def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       internal_refs = locate_internal_refs
-      @files.each do |identifier, x|
+      @files.each_with_index do |(identifier, x), i|
+        i.positive? && Array(@directives).include?("bare-after-first") and
+          @compile_options.merge!(bare: true)
         if x[:attachment] then copy_file_to_dest(x)
         else
-          file, filename = targetfile(x, true)
+          file, filename = targetfile(x, read: true)
           file = update_xrefs(file, identifier, internal_refs)
           Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
             f.write(file)

data/lib/metanorma/collection_manifest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "util"
+
 module Metanorma
   # Metanorma collection's manifest
   class CollectionManifest
@@ -15,6 +17,7 @@ module Metanorma
       @title = title
       @docref = docref
       @manifest = manifest
+      @disambig = Util::DisambigFiles.new
     end
 
     class << self
@@ -43,9 +46,11 @@ module Metanorma
       # @return [Hash{String=>String}]
       def parse_docref(mnf)
         mnf.xpath("xmlns:docref").map do |dr|
-          h = { "identifier" => dr.at("identifier").text }
-          h["fileref"] = dr[:fileref] if dr[:fileref]
+          h = { "identifier" => dr.at("identifier").children.to_xml }
+          dr[:fileref] and h["fileref"] = dr[:fileref]
           h["attachment"] = dr[:attachment] if dr[:attachment]
+          h["sectionsplit"] = dr[:sectionsplit] if dr[:sectionsplit]
+          h["presentation-xml"] = dr[:presentationxml] if dr[:presentationxml]
           h
         end
       end
@@ -57,18 +62,19 @@ module Metanorma
       @manifest.each { |mnf| mnf.collection = col }
     end
 
-    # @param dir [String] path to coolection
+    # @param dir [String] path to collection
     # @return [Hash<String, Metanorma::Document>]
     def documents(dir = "")
       docs = @docref.each_with_object({}) do |dr, m|
         next m unless dr["fileref"]
 
-        m[dr["identifier"]] = Document.parse_file(File.join(dir, dr["fileref"]), dr["attachment"])
+        m[dr["identifier"]] = Document.parse_file(
+          File.join(dir, dr["fileref"]),
+          dr["attachment"], dr["identifier"]
+        )
         m
       end
-      @manifest.reduce(docs) do |mem, mnf|
-        mem.merge mnf.documents(dir)
-      end
+      @manifest.reduce(docs) { |mem, mnf| mem.merge mnf.documents(dir) }
     end
 
     # @param builder [Nokogiri::XML::Builder]
@@ -99,14 +105,28 @@ module Metanorma
 
     # @param builder [Nokogiri::XML::Builder]
     def docref_to_xml(builder)
+      @disambig = Util::DisambigFiles.new
       @docref.each do |dr|
-        drf = builder.docref { |b| b.identifier dr["identifier"] }
-        drf[:fileref] = dr["fileref"]
-        drf[:attachment] = dr["attachment"] if dr["attachment"]
-        if collection.directives.include?("documents-inline")
-          id = collection.documents.find_index { |k, _| k == dr["identifier"] }
-          drf[:id] = format("doc%<index>09d", index: id)
+        drf = builder.docref do |b|
+          b.identifier do |i|
+            i << dr["identifier"]
+          end
+        end
+        docref_to_xml_attrs(drf, dr)
+      end
+    end
+
+    def docref_to_xml_attrs(elem, docref)
+      elem[:fileref] = @disambig.source2dest_filename(docref["fileref"])
+      elem[:attachment] = docref["attachment"] if docref["attachment"]
+      elem[:sectionsplit] = docref["sectionsplit"] if docref["sectionsplit"]
+      elem[:presentationxml] = "true" if docref["presentation-xml"] &&
+        docref["presentation-xml"] == "true" || docref["presentation-xml"] == true
+      if collection.directives.include?("documents-inline")
+        id = collection.documents.find_index do |k, _|
+          k == docref["identifier"]
         end
+        elem[:id] = format("doc%<index>09d", index: id)
       end
     end
   end

data/lib/metanorma/collection_renderer.rb

@@ -3,6 +3,7 @@
 require "isodoc"
 require_relative "collection_fileprocess"
 require_relative "fontist_utils"
+require_relative "util"
 
 module Metanorma
   # XML collection renderer
@@ -21,9 +22,9 @@ module Metanorma
     # the collection, and that the flavour gem can sensibly process it. We may
     # need to enhance metadata in the flavour gems isodoc/metadata.rb with
     # collection metadata
-    def initialize(xml, folder, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def initialize(collection, folder, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       check_options options
-      @xml = Nokogiri::XML xml # @xml is the collection manifest
+      @xml = Nokogiri::XML collection.to_xml # @xml is the collection manifest
       @lang = @xml&.at(ns("//bibdata/language"))&.text || "en"
       @script = @xml&.at(ns("//bibdata/script"))&.text || "Latn"
       @doctype = doctype
@@ -34,12 +35,16 @@ module Metanorma
 
       @outdir = options[:output_folder]
       @coverpage = options[:coverpage]
-      @format = options[:format]
+      @format = Util.sort_extensions_execution(options[:format])
       @compile_options = options[:compile] || {}
       @log = options[:log]
+      @documents = collection.documents
+      @directives = collection.directives
+      @disambig = Util::DisambigFiles.new
 
       # list of files in the collection
       @files = read_files folder
+      isodoc_populate(@isodoc)
       FileUtils.rm_rf @outdir
       FileUtils.mkdir_p @outdir
     end
@@ -51,7 +56,7 @@ module Metanorma
     # @option options [Strong] :ourput_folder output directory
     def self.render(col, options = {})
       folder = File.dirname col.file
-      cr = new(col.to_xml, folder, options)
+      cr = new(col, folder, options)
       cr.files
       cr.concatenate(col, options)
       cr.coverpage if options[:format]&.include?(:html)
@@ -61,11 +66,13 @@ module Metanorma
       options[:format] << :presentation if options[:format].include?(:pdf)
       options[:format].uniq.each do |e|
         next unless %i(presentation xml).include?(e)
+
         ext = e == :presentation ? "presentation.xml" : e.to_s
         out = col.clone
         out.directives << "documents-inline"
-        out.documents.keys.each do |id|
+        out.documents.each_key do |id|
           next if @files[id][:attachment]
+
           filename = @files[id][:outputs][e]
           out.documents[id] = Metanorma::Document.raw_file(filename)
         end
@@ -100,15 +107,21 @@ module Metanorma
     end
 
     # The isodoc class for the metanorma flavour we are using
-    def isodoc # rubocop:disable Metrics/MethodLength
+    def isodoc
       x = Asciidoctor.load nil, backend: @doctype.to_sym
       isodoc = x.converter.html_converter(Dummy.new)
       isodoc.i18n_init(@lang, @script) # read in internationalisation
+      isodoc.metadata_init(@lang, @script, isodoc.i18n)
+      isodoc.info(@xml, nil)
+      isodoc
+    end
+
+    def isodoc_populate(isodoc)
       # create the @meta class of isodoc, with "navigation" set to the index bar
       # extracted from the manifest
       nav = indexfile(@xml.at(ns("//manifest")))
       i18n = isodoc.i18n
-      i18n.set(:navigation, nav)
+      i18n.set("navigation", nav)
       isodoc.metadata_init(@lang, @script, i18n)
       # populate the @meta class of isodoc with the various metadata fields
       # native to the flavour; used to populate Liquid
@@ -135,6 +148,8 @@ module Metanorma
     # populate liquid template of ARGV[1] with metadata extracted from
     # collection manifest
     def coverpage
+      return unless @coverpage
+
       File.open(File.join(@outdir, "index.html"), "w:UTF-8") do |f|
         f.write @isodoc.populate_template(File.read(@coverpage))
       end
@@ -163,11 +178,19 @@ module Metanorma
     # @param builder [Nokogiri::XML::Builder]
     def docrefs(elm, builder)
       elm.xpath(ns("./docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
-        link = if d["fileref"] then d["fileref"].sub(/\.xml$/, ".html")
-               else d["id"] + ".html"
-               end
-        builder.li { builder.a identifier, href: link }
+        ident = d.at(ns("./identifier")).children.to_xml
+        builder.li do |li|
+          li.a **{ href: index_link(d, ident) } do |a|
+            a << ident
+          end
+        end
+      end
+    end
+
+    def index_link(docref, ident)
+      if docref["fileref"]
+        @files[ident][:out_path].sub(/\.xml$/, ".html")
+      else "#{docref['id']}.html"
       end
     end
 
@@ -193,14 +216,20 @@ module Metanorma
 
     private
 
+    def format_sort(formats)
+      ret = []
+      formats.include?(:xml) and ret << :xml
+      formats.include?(:presentation) and ret << :presentation
+      a = %i(presentation xml)
+      ret + formats.reject { |i| a.include? i }
+    end
+
     # @param options [Hash]
     # @raise [ArgumentError]
     def check_options(options)
       unless options[:format].is_a?(Array) && (FORMATS & options[:format]).any?
         raise ArgumentError, "Need to specify formats (xml,html,pdf,doc)"
       end
-      return if !options[:format].include?(:html) || options[:coverpage]
-      raise ArgumentError, "Need to specify a coverpage to render HTML"
     end
   end
 end

data/lib/metanorma/compile.rb

@@ -6,6 +6,7 @@ require "fontist"
 require "fontist/manifest/install"
 require_relative "compile_validate"
 require_relative "fontist_utils"
+require_relative "util"
 
 module Metanorma
   class Compile
@@ -20,7 +21,7 @@ module Metanorma
     def compile(filename, options = {})
       require_libraries(options)
       options = options_extract(filename, options)
-      validate_type(options) && validate_format(options) || (return nil)
+      validate_options(options)
       @processor = @registry.find_processor(options[:type].to_sym)
       extensions = get_extensions(options) or return nil
       (file, isodoc = process_input(filename, options)) or return nil
@@ -35,7 +36,7 @@ module Metanorma
     end
 
     def xml_options_extract(file)
-      xml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
       if xml.root
         @registry.root_tags.each do |k, v|
           return { type: k } if v == xml.root.name
@@ -74,7 +75,9 @@ module Metanorma
           memo
         end
       end
-      if !extensions.include?(:presentation) and extensions.any? { |e| @processor.use_presentation_xml(e) }
+      if !extensions.include?(:presentation) && extensions.any? do |e|
+        @processor.use_presentation_xml(e)
+      end
         extensions << :presentation
       end
       extensions
@@ -88,7 +91,7 @@ module Metanorma
         options[:asciimath] and
           file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
         dir = File.dirname(filename)
-        dir != '.' and
+        dir != "." and
           file.gsub!(/^include::/, "include::#{dir}/")
         [file, @processor.input_to_isodoc(file, filename, options)]
       when ".xml"
@@ -108,15 +111,17 @@ module Metanorma
 
     def relaton_export(isodoc, options)
       return unless options[:relaton]
-      xml = Nokogiri::XML(isodoc)
+
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
-      #docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
-      #outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
+      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
+      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
       File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
     end
 
     def clean_sourcecode(xml)
-      xml.xpath(".//callout | .//annotation | .//xmlns:callout | .//xmlns:annotation").each do |x|
+      xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
+                ".//xmlns:annotation").each do |x|
         x.remove
       end
       xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
@@ -125,12 +130,13 @@ module Metanorma
 
     def extract(isodoc, dirname, extract_types)
       return unless dirname
+
       if extract_types.nil? || extract_types.empty?
         extract_types = [:sourcecode, :image, :requirement]
       end
       FileUtils.rm_rf dirname
       FileUtils.mkdir_p dirname
-      xml = Nokogiri::XML(isodoc)
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
       image_export(xml, dirname) if extract_types.include? :image
       requirement_export(xml, dirname) if extract_types.include? :requirement
@@ -176,17 +182,6 @@ module Metanorma
       end
     end
 
-    # dependency ordering
-    def sort_extensions_execution(ext)
-      case ext
-      when :xml then 0
-      when :rxl then 1
-      when :presentation then 2
-      else
-        99
-      end
-    end
-
     def wrap_html(options, file_extension, outfilename)
       if options[:wrapper] && /html$/.match(file_extension)
         outfilename = outfilename.sub(/\.html$/, "")
@@ -201,34 +196,22 @@ module Metanorma
       f = change_output_dir options
       xml_name = f.sub(/\.[^.]+$/, ".xml")
       presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
-      extensions.sort do |a, b|
-        sort_extensions_execution(a) <=> sort_extensions_execution(b)
-      end.each do |ext|
-        isodoc_options = @processor.extract_options(file)
-        isodoc_options[:datauriimage] = true if options[:datauriimage]
-        isodoc_options[:sourcefilename] = options[:filename]
+      Util.sort_extensions_execution(extensions).each do |ext|
         file_extension = @processor.output_formats[ext]
         outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        if ext == :pdf
-          font_locations = FontistUtils.fontist_font_locations(@processor, options)
-          font_locations and
-            isodoc_options[:mn2pdf] = { font_manifest_file: font_locations.path }
-        end
+        isodoc_options = get_isodoc_options(file, options, ext)
         if ext == :rxl
           options[:relaton] = outfilename
           relaton_export(isodoc, options)
+        elsif options[:passthrough_presentation_xml] && ext == :presentation
+          FileUtils.cp f, presentationxml_name
         else
           begin
             @processor.use_presentation_xml(ext) ?
               @processor.output(nil, presentationxml_name, outfilename, ext, isodoc_options) :
               @processor.output(isodoc, xml_name, outfilename, ext, isodoc_options)
           rescue StandardError => e
-            if e.message.start_with? "mn2pdf"
-              @errors << e.message
-            else
-              puts e.message
-              puts e.backtrace.join("\n")
-            end
+            isodoc_error_process(e)
           end
         end
         wrap_html(options, file_extension, outfilename)
@@ -237,6 +220,28 @@ module Metanorma
 
     private
 
+    def isodoc_error_process(err)
+      if err.message.include? "Fatal:"
+        @errors << err.message
+      else
+        puts err.message
+        puts err.backtrace.join("\n")
+      end
+    end
+
+    def get_isodoc_options(file, options, ext)
+      isodoc_options = @processor.extract_options(file)
+      isodoc_options[:datauriimage] = true if options[:datauriimage]
+      isodoc_options[:sourcefilename] = options[:filename]
+      isodoc_options[:bare] = options[:bare]
+      isodoc_options[:sectionsplit] = options[:sectionsplit]
+      if ext == :pdf
+        floc = FontistUtils.fontist_font_locations(@processor, options) and
+          isodoc_options[:mn2pdf] = { font_manifest_file: floc.path }
+      end
+      isodoc_options
+    end
+
     # @param options [Hash]
     # @return [String]
     def change_output_dir(options)