metanorma 1.3.4 → 1.3.8

data/lib/metanorma/collection_manifest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "util"
+
 module Metanorma
   # Metanorma collection's manifest
   class CollectionManifest
@@ -15,6 +17,7 @@ module Metanorma
       @title = title
       @docref = docref
       @manifest = manifest
+      @disambig = Util::DisambigFiles.new
     end
 
     class << self
@@ -43,9 +46,11 @@ module Metanorma
       # @return [Hash{String=>String}]
       def parse_docref(mnf)
         mnf.xpath("xmlns:docref").map do |dr|
-          h = { "identifier" => dr.at("identifier").text }
-          dr[:fileref] and h["fileref"] = dr[:fileref]
-          h["attachment"] = dr[:attachment] if dr[:attachment]
+          h = { "identifier" => dr.at("identifier").children.to_xml }
+          %i(fileref attachment sectionsplit index).each do |s|
+            h[s.to_s] = dr[s] if dr[s]
+          end
+          h["presentation-xml"] = dr[:presentationxml] if dr[:presentationxml]
           h
         end
       end
@@ -57,7 +62,7 @@ module Metanorma
       @manifest.each { |mnf| mnf.collection = col }
     end
 
-    # @param dir [String] path to coolection
+    # @param dir [String] path to collection
     # @return [Hash<String, Metanorma::Document>]
     def documents(dir = "")
       docs = @docref.each_with_object({}) do |dr, m|
@@ -65,7 +70,7 @@ module Metanorma
 
         m[dr["identifier"]] = Document.parse_file(
           File.join(dir, dr["fileref"]),
-          dr["attachment"], dr["identifier"]
+          dr["attachment"], dr["identifier"], dr["index"]
         )
         m
       end
@@ -100,14 +105,34 @@ module Metanorma
 
     # @param builder [Nokogiri::XML::Builder]
     def docref_to_xml(builder)
+      @disambig = Util::DisambigFiles.new
       @docref.each do |dr|
-        drf = builder.docref { |b| b.identifier dr["identifier"] }
-        drf[:fileref] = Util::source2dest_filename(dr["fileref"])
-        drf[:attachment] = dr["attachment"] if dr["attachment"]
-        if collection.directives.include?("documents-inline")
-          id = collection.documents.find_index { |k, _| k == dr["identifier"] }
-          drf[:id] = format("doc%<index>09d", index: id)
+        drf = builder.docref do |b|
+          b.identifier do |i|
+            i << dr["identifier"]
+          end
+        end
+        docref_to_xml_attrs(drf, dr)
+      end
+    end
+
+    def docref_to_xml_attrs(elem, docref)
+      elem[:fileref] = @disambig.source2dest_filename(docref["fileref"])
+      %i(attachment sectionsplit).each do |i|
+        elem[i] = docref[i.to_s] if docref[i.to_s]
+      end
+      elem[:index] = docref.has_key?("index") ? docref["index"] : "true"
+      elem[:presentationxml] = "true" if docref["presentation-xml"] &&
+        [true, "true"].include?(docref["presentation-xml"])
+      docref_to_xml_attrs_id(elem, docref)
+    end
+
+    def docref_to_xml_attrs_id(elem, docref)
+      if collection.directives.include?("documents-inline")
+        id = collection.documents.find_index do |k, _|
+          k == docref["identifier"]
         end
+        elem[:id] = format("doc%<index>09d", index: id)
       end
     end
   end

data/lib/metanorma/collection_renderer.rb

@@ -3,6 +3,7 @@
 require "isodoc"
 require_relative "collection_fileprocess"
 require_relative "fontist_utils"
+require_relative "util"
 
 module Metanorma
   # XML collection renderer
@@ -34,13 +35,18 @@ module Metanorma
 
       @outdir = options[:output_folder]
       @coverpage = options[:coverpage]
-      @format = options[:format]
+      @format = Util.sort_extensions_execution(options[:format])
       @compile_options = options[:compile] || {}
+      @compile_options[:no_install_fonts] = true if options[:no_install_fonts]
       @log = options[:log]
       @documents = collection.documents
+      @directives = collection.directives
+      @disambig = Util::DisambigFiles.new
+      @compile = Compile.new
 
       # list of files in the collection
       @files = read_files folder
+      isodoc_populate(@isodoc)
       FileUtils.rm_rf @outdir
       FileUtils.mkdir_p @outdir
     end
@@ -48,14 +54,20 @@ module Metanorma
     # @param col [Metanorma::Collection] XML collection
     # @param options [Hash]
     # @option options [String] :coverpage cover page HTML (Liquid template)
-    # @option options [Array<Synbol>] :format list of formats
+    # @option options [Array<Symbol>] :format list of formats
     # @option options [Strong] :ourput_folder output directory
     def self.render(col, options = {})
       folder = File.dirname col.file
+      # require "byebug"; byebug
+      warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
       cr = new(col, folder, options)
+      warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"
       cr.files
+      warn "\n\n\n\n\nConcatenate: #{DateTime.now.strftime('%H:%M:%S')}"
       cr.concatenate(col, options)
+      warn "\n\n\n\n\nCoverpage: #{DateTime.now.strftime('%H:%M:%S')}"
       cr.coverpage if options[:format]&.include?(:html)
+      warn "\n\n\n\n\nDone: #{DateTime.now.strftime('%H:%M:%S')}"
     end
 
     def concatenate(col, options)
@@ -66,13 +78,15 @@ module Metanorma
         ext = e == :presentation ? "presentation.xml" : e.to_s
         out = col.clone
         out.directives << "documents-inline"
-        out.documents.keys.each do |id|
-          next if @files[id][:attachment]
+        out.documents.each_key do |id|
+          next if @files[id][:attachment] || @files[id][:outputs].nil?
 
           filename = @files[id][:outputs][e]
           out.documents[id] = Metanorma::Document.raw_file(filename)
         end
-        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") { |f| f.write(out.to_xml) }
+        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") do |f|
+          f.write(out.to_xml)
+        end
       end
       options[:format].include?(:pdf) and
         pdfconv.convert(File.join(@outdir, "collection.presentation.xml"))
@@ -103,15 +117,21 @@ module Metanorma
     end
 
     # The isodoc class for the metanorma flavour we are using
-    def isodoc # rubocop:disable Metrics/MethodLength
+    def isodoc
       x = Asciidoctor.load nil, backend: @doctype.to_sym
       isodoc = x.converter.html_converter(Dummy.new)
       isodoc.i18n_init(@lang, @script) # read in internationalisation
+      isodoc.metadata_init(@lang, @script, isodoc.i18n)
+      isodoc.info(@xml, nil)
+      isodoc
+    end
+
+    def isodoc_populate(isodoc)
       # create the @meta class of isodoc, with "navigation" set to the index bar
       # extracted from the manifest
       nav = indexfile(@xml.at(ns("//manifest")))
       i18n = isodoc.i18n
-      i18n.set(:navigation, nav)
+      i18n.set("navigation", nav)
       isodoc.metadata_init(@lang, @script, i18n)
       # populate the @meta class of isodoc with the various metadata fields
       # native to the flavour; used to populate Liquid
@@ -159,7 +179,7 @@ module Metanorma
     # @param elm [Nokogiri::XML::Element]
     # @param builder [Nokogiri::XML::Builder]
     def indexfile_docref(elm, builder)
-      return "" unless elm.at(ns("./docref"))
+      return "" unless elm.at(ns("./docref[@index = 'true']"))
 
       builder.ul { |b| docrefs(elm, b) }
     end
@@ -167,12 +187,20 @@ module Metanorma
     # @param elm [Nokogiri::XML::Element]
     # @param builder [Nokogiri::XML::Builder]
     def docrefs(elm, builder)
-      elm.xpath(ns("./docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
-        link = if d["fileref"] then d["fileref"].sub(/\.xml$/, ".html")
-               else d["id"] + ".html"
-               end
-        builder.li { builder.a identifier, href: link }
+      elm.xpath(ns("./docref[@index = 'true']")).each do |d|
+        ident = d.at(ns("./identifier")).children.to_xml
+        builder.li do |li|
+          li.a **{ href: index_link(d, ident) } do |a|
+            a << ident
+          end
+        end
+      end
+    end
+
+    def index_link(docref, ident)
+      if docref["fileref"]
+        @files[ident][:out_path].sub(/\.xml$/, ".html")
+      else "#{docref['id']}.html"
       end
     end
 
@@ -198,6 +226,14 @@ module Metanorma
 
     private
 
+    def format_sort(formats)
+      ret = []
+      formats.include?(:xml) and ret << :xml
+      formats.include?(:presentation) and ret << :presentation
+      a = %i(presentation xml)
+      ret + formats.reject { |i| a.include? i }
+    end
+
     # @param options [Hash]
     # @raise [ArgumentError]
     def check_options(options)

data/lib/metanorma/compile.rb

@@ -6,6 +6,8 @@ require "fontist"
 require "fontist/manifest/install"
 require_relative "compile_validate"
 require_relative "fontist_utils"
+require_relative "util"
+require_relative "sectionsplit"
 
 module Metanorma
   class Compile
@@ -15,6 +17,8 @@ module Metanorma
     def initialize
       @registry = Metanorma::Registry.instance
       @errors = []
+      @isodoc = IsoDoc::Convert.new({})
+      @fontist_installed = false
     end
 
     def compile(filename, options = {})
@@ -26,7 +30,8 @@ module Metanorma
       (file, isodoc = process_input(filename, options)) or return nil
       relaton_export(isodoc, options)
       extract(isodoc, options[:extract], options[:extract_type])
-      FontistUtils.install_fonts(@processor, options)
+      FontistUtils.install_fonts(@processor, options) unless @fontist_installed
+      @fontist_installed = true
       process_extensions(extensions, file, isodoc, options)
     end
 
@@ -35,7 +40,7 @@ module Metanorma
     end
 
     def xml_options_extract(file)
-      xml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
       if xml.root
         @registry.root_tags.each do |k, v|
           return { type: k } if v == xml.root.name
@@ -53,7 +58,7 @@ module Metanorma
       dir = filename.sub(%r(/[^/]+$), "/")
       options[:relaton] ||= "#{dir}/#{o[:relaton]}" if o[:relaton]
       options[:sourcecode] ||= "#{dir}/#{o[:sourcecode]}" if o[:sourcecode]
-      options[:extension_keys] ||= o[:extensions]&.split(/,[ ]*/)&.map(&:to_sym)
+      options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
       options[:extension_keys] = nil if options[:extension_keys] == [:all]
       options[:format] ||= :asciidoc
       options[:filename] = filename
@@ -61,12 +66,10 @@ module Metanorma
     end
 
     def get_extensions(options)
-      options[:extension_keys] ||= @processor.output_formats.reduce([]) do |memo, (k, _)|
-        memo << k
-      end
+      options[:extension_keys] ||=
+        @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
       extensions = options[:extension_keys].reduce([]) do |memo, e|
-        if @processor.output_formats[e]
-          memo << e
+        if @processor.output_formats[e] then memo << e
         else
           message = "[metanorma] Error: #{e} format is not supported for this standard."
           @errors << message
@@ -74,7 +77,9 @@ module Metanorma
           memo
         end
       end
-      if !extensions.include?(:presentation) and extensions.any? { |e| @processor.use_presentation_xml(e) }
+      if !extensions.include?(:presentation) && extensions.any? do |e|
+        @processor.use_presentation_xml(e)
+      end
         extensions << :presentation
       end
       extensions
@@ -82,41 +87,50 @@ module Metanorma
 
     def process_input(filename, options)
       case extname = File.extname(filename)
-      when ".adoc"
-        Util.log("[metanorma] Processing: AsciiDoc input.", :info)
-        file = read_file(filename)
-        options[:asciimath] and
-          file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
-        dir = File.dirname(filename)
-        dir != '.' and
-          file.gsub!(/^include::/, "include::#{dir}/")
-        [file, @processor.input_to_isodoc(file, filename, options)]
-      when ".xml"
-        Util.log("[metanorma] Processing: Metanorma XML input.", :info)
-        # TODO NN: this is a hack -- we should provide/bridge the
-        # document attributes in Metanorma XML
-        ["", read_file(filename)]
+      when ".adoc" then process_input_adoc(filename, options)
+      when ".xml" then process_input_xml(filename, options)
       else
-        Util.log("[metanorma] Error: file extension #{extname} is not supported.", :error)
+        Util.log("[metanorma] Error: file extension #{extname} "\
+                 "is not supported.", :error)
         nil
       end
     end
 
+    def process_input_adoc(filename, options)
+      Util.log("[metanorma] Processing: AsciiDoc input.", :info)
+      file = read_file(filename)
+      options[:asciimath] and
+        file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
+      dir = File.dirname(filename)
+      dir != "." and
+        file.gsub!(/^include::/, "include::#{dir}/")
+      [file, @processor.input_to_isodoc(file, filename, options)]
+    end
+
+    def process_input_xml(filename, _options)
+      Util.log("[metanorma] Processing: Metanorma XML input.", :info)
+      # TODO NN: this is a hack -- we should provide/bridge the
+      # document attributes in Metanorma XML
+      ["", read_file(filename)]
+    end
+
     def read_file(filename)
       File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
     end
 
     def relaton_export(isodoc, options)
       return unless options[:relaton]
-      xml = Nokogiri::XML(isodoc)
+
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
-      #docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
-      #outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
+      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
+      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
       File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
     end
 
     def clean_sourcecode(xml)
-      xml.xpath(".//callout | .//annotation | .//xmlns:callout | .//xmlns:annotation").each do |x|
+      xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
+                ".//xmlns:annotation").each do |x|
         x.remove
       end
       xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
@@ -125,12 +139,13 @@ module Metanorma
 
     def extract(isodoc, dirname, extract_types)
       return unless dirname
+
       if extract_types.nil? || extract_types.empty?
-        extract_types = [:sourcecode, :image, :requirement]
+        extract_types = %i[sourcecode image requirement]
       end
       FileUtils.rm_rf dirname
       FileUtils.mkdir_p dirname
-      xml = Nokogiri::XML(isodoc)
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
       image_export(xml, dirname) if extract_types.include? :image
       requirement_export(xml, dirname) if extract_types.include? :requirement
@@ -151,7 +166,7 @@ module Metanorma
       xml.at("//image | //xmlns:image") or return
       FileUtils.mkdir_p "#{dirname}/image"
       xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
-        next unless /^data:image/.match s["src"]
+        next unless /^data:image/.match? s["src"]
 
         %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
         filename = s["filename"] || sprintf("image-%04d.%s", i, imgtype)
@@ -176,17 +191,6 @@ module Metanorma
       end
     end
 
-    # dependency ordering
-    def sort_extensions_execution(ext)
-      case ext
-      when :xml then 0
-      when :rxl then 1
-      when :presentation then 2
-      else
-        99
-      end
-    end
-
     def wrap_html(options, file_extension, outfilename)
       if options[:wrapper] && /html$/.match(file_extension)
         outfilename = outfilename.sub(/\.html$/, "")
@@ -201,34 +205,25 @@ module Metanorma
       f = change_output_dir options
       xml_name = f.sub(/\.[^.]+$/, ".xml")
       presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
-      extensions.sort do |a, b|
-        sort_extensions_execution(a) <=> sort_extensions_execution(b)
-      end.each do |ext|
-        isodoc_options = @processor.extract_options(file)
-        isodoc_options[:datauriimage] = true if options[:datauriimage]
-        isodoc_options[:sourcefilename] = options[:filename]
+      Util.sort_extensions_execution(extensions).each do |ext|
         file_extension = @processor.output_formats[ext]
         outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        if ext == :pdf
-          font_locations = FontistUtils.fontist_font_locations(@processor, options)
-          font_locations and
-            isodoc_options[:mn2pdf] = { font_manifest_file: font_locations.path }
-        end
+        isodoc_options = get_isodoc_options(file, options, ext)
         if ext == :rxl
-          options[:relaton] = outfilename
-          relaton_export(isodoc, options)
+          relaton_export(isodoc, options.merge(relaton: outfilename))
+        elsif options[:passthrough_presentation_xml] && ext == :presentation
+          FileUtils.cp f, presentationxml_name
+        elsif ext == :html && options[:sectionsplit]
+          sectionsplit_convert(xml_name, isodoc, outfilename, isodoc_options)
         else
           begin
-            @processor.use_presentation_xml(ext) ?
-              @processor.output(nil, presentationxml_name, outfilename, ext, isodoc_options) :
-              @processor.output(isodoc, xml_name, outfilename, ext, isodoc_options)
-          rescue StandardError => e
-            if e.message.include? "Fatal:"
-              @errors << e.message
+            if @processor.use_presentation_xml(ext)
+              @processor.output(nil, presentationxml_name, outfilename, ext, isodoc_options)
             else
-              puts e.message
-              puts e.backtrace.join("\n")
+              @processor.output(isodoc, xml_name, outfilename, ext, isodoc_options)
             end
+          rescue StandardError => e
+            isodoc_error_process(e)
           end
         end
         wrap_html(options, file_extension, outfilename)
@@ -237,6 +232,29 @@ module Metanorma
 
     private
 
+    def isodoc_error_process(err)
+      if err.message.include? "Fatal:"
+        @errors << err.message
+      else
+        puts err.message
+        puts err.backtrace.join("\n")
+      end
+    end
+
+    def get_isodoc_options(file, options, ext)
+      isodoc_options = @processor.extract_options(file)
+      isodoc_options[:datauriimage] = true if options[:datauriimage]
+      isodoc_options[:sourcefilename] = options[:filename]
+      %i(bare sectionsplit no_install_fonts).each do |x|
+        isodoc_options[x] ||= options[x]
+      end
+      if ext == :pdf
+        floc = FontistUtils.fontist_font_locations(@processor, options) and
+          isodoc_options[:mn2pdf] = { font_manifest_file: floc.path }
+      end
+      isodoc_options
+    end
+
     # @param options [Hash]
     # @return [String]
     def change_output_dir(options)