metanorma 1.3.3 → 1.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fec4f760c22a19793bca3f3cb2c5abc0b355a8554d5ac0720fe8e679a26db842
-  data.tar.gz: ffa03687166474e66611a2fa0f5841281ae5b1127e078dd7cbf2993ec1a15f65
+  metadata.gz: 16196ab7719ac294eb22120afeb64ebceee75e83469bf0a4ae1abd2221952139
+  data.tar.gz: d1ed0d591efa82eb1e40b09f6598f98f6d7738435c85b6146e2ec7ada41a3413
 SHA512:
-  metadata.gz: 9721c89fe96d4bfb1bf6458fc79d824fa16887a349b2bdcd9618d6528ec0b32abf86158bc7658de8288bcf130ce3008427fcc06fe036179c652b70a2a3edd9f6
-  data.tar.gz: 74993bec3d7dca356699d821dadb55f35f7f7c20d6a8270e2c3fc9e224225504fe4fe3dee77cdf9cbc0a55eeee033df0253a17307fac8fec76772dc02c5c2f2f
+  metadata.gz: 4618e5ebe7f688ae9eec5680d0f17aa2795391e1485e749f975052952378742a0c29be722e46aef10c34432a658294bb06c5eac945c56a27c65c8ab7aedc944b
+  data.tar.gz: e0b3b3ac8846b3d6ecd1fb75691d525f3763c126d2c1db3469c1192c93a14f9a168ec75f2a851f92260334c514704072cc9a59c360034d8de578ca37a2bebf24

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/.rubocop.yml

@@ -7,4 +7,4 @@ inherit_from:
 # ...
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5

data/Gemfile

@@ -6,6 +6,6 @@ git_source(:github) { |repo| "https://github.com/#{repo}" }
 
 gemspec
 
-if File.exist? 'Gemfile.devel'
-  eval File.read('Gemfile.devel'), nil, 'Gemfile.devel' # rubocop:disable Security/Eval
+if File.exist? "Gemfile.devel"
+  eval File.read("Gemfile.devel"), nil, "Gemfile.devel" # rubocop:disable Security/Eval
 end

data/lib/metanorma/collection.rb

@@ -4,6 +4,7 @@ require "relaton"
 require "relaton/cli"
 require "metanorma/collection_manifest"
 require "metanorma-utils"
+require_relative "util"
 
 module Metanorma
   # Metanorma collection of documents
@@ -18,6 +19,8 @@ module Metanorma
     # @return [Hash<String, Metanorma::Document>]
     attr_accessor :documents
 
+    attr_accessor :disambig
+
     # @param file [String] path to source file
     # @param directives [Array<String>] documents-inline to inject the XML into
     #   the collection manifest; documents-external to keeps them outside
@@ -41,6 +44,7 @@ module Metanorma
       @prefatory = args[:prefatory]
       @final = args[:final]
       @log = Metanorma::Utils::Log.new
+      @disambig = Util::DisambigFiles.new
     end
 
     # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
@@ -87,7 +91,9 @@ module Metanorma
       private
 
       def parse_xml(file)
-        xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
+        xml = Nokogiri::XML File.read(file, encoding: "UTF-8") do |config|
+          config.huge
+        end
         if (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata"))
           bd = Relaton::Cli.parse_xml b
         end

data/lib/metanorma/collection_fileparse.rb

@@ -11,15 +11,21 @@ module Metanorma
       xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n, {})
       xrefs.parse xml
       xrefs.get.each_with_object({}) do |(k, v), ret|
-        ret[v[:type]] ||= {}
-        index = if v[:container] || v[:label].nil? || v[:label].empty?
-                  UUIDTools::UUID.random_create.to_s
-                else v[:label]
-                end
-        ret[v[:type]][index] = k
+        read_anchors1(k, v, ret)
       end
     end
 
+    def read_anchors1(key, val, ret)
+      val[:type] ||= "clause"
+      ret[val[:type]] ||= {}
+      index = if val[:container] || val[:label].nil? || val[:label].empty?
+                UUIDTools::UUID.random_create.to_s
+              else val[:label]
+              end
+      ret[val[:type]][index] = key
+      ret[val[:type]][val[:value]] = key if val[:value]
+    end
+
     # @param id [String]
     # @param read [Boolean]
     # @return [Array<String, nil>]
@@ -31,29 +37,36 @@ module Metanorma
 
     # @param bib [Nokogiri::XML::Element]
     # @param identifier [String]
-    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      docid = bib&.at(ns("./docidentifier"))&.text
-      unless @files[docid]
-        error = "[metanorma] Cannot find crossreference to document #{docid} "\
-          "in document #{identifier}."
-        @log.add("Cross-References", nil, error)
-        Util.log(error, :warning)
-        return
-      end
-      id = bib["id"]
-      newbib = bib.replace(@files[docid][:bibdata])
-      newbib.name = "bibitem"
-      newbib["id"] = id
-      newbib["hidden"] = "true"
-      newbib&.at(ns("./ext"))&.remove
+    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize
+      docid = bib&.at(ns("./docidentifier"))&.children&.to_xml
+      return fail_update_bibitem(docid, identifier) unless @files[docid]
+
+      newbib = dup_bibitem(docid, bib)
+      bib.replace(newbib)
       _file, url = targetfile(@files[docid], relative: true, read: false,
-                                             doc: !@files[docid][:attachment])
+                              doc: !@files[docid][:attachment])
       uri_node = Nokogiri::XML::Node.new "uri", newbib
       uri_node[:type] = "citation"
       uri_node.content = url
       newbib.at(ns("./docidentifier")).previous = uri_node
     end
 
+    def fail_update_bibitem(docid, identifier)
+      error = "[metanorma] Cannot find crossreference to document #{docid} "\
+        "in document #{identifier}."
+      @log.add("Cross-References", nil, error)
+      Util.log(error, :warning)
+    end
+
+    def dup_bibitem(docid, bib)
+      newbib = @files[docid][:bibdata].dup
+      newbib.name = "bibitem"
+      newbib["hidden"] = "true"
+      newbib&.at("./*[local-name() = 'ext']")&.remove
+      newbib["id"] = bib["id"]
+      newbib
+    end
+
     # Resolves direct links to other files in collection
     # (repo(current-metanorma-collection/x),
     # and indirect links to other files in collection
@@ -64,12 +77,12 @@ module Metanorma
     # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to anchor to filename
     # @return [String] XML content
     def update_xrefs(file, identifier, internal_refs)
-      docxml = Nokogiri::XML(file)
+      docxml = Nokogiri::XML(file) { |config| config.huge }
       update_indirect_refs_to_docs(docxml, internal_refs)
       add_document_suffix(identifier, docxml)
       update_direct_refs_to_docs(docxml, identifier)
       svgmap_resolve(datauri_encode(docxml))
-      docxml.xpath(ns("//references[not(./bibitem[not(@hidden) or "\
+      docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or "\
                       "@hidden = 'false'])]")).each do |f|
         f["hidden"] = "true"
       end
@@ -103,12 +116,18 @@ module Metanorma
     # Preferably with anchor, and is a job to realise dynamic lookup
     # of localities.
     def update_direct_refs_to_docs(docxml, identifier)
+      erefs = docxml.xpath(ns("//eref"))
+        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
+        m[:citeas][i["citeas"]] = true
+        m[:bibitemid][i["bibitemid"]] = true
+      end
       docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
         docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
         next unless docid && %r{^current-metanorma-collection/}.match(docid)
 
         update_bibitem(b, identifier)
-        update_anchors(b, docxml, docid)
+        docid = b&.at(ns("./docidentifier"))&.children&.to_xml or next
+        erefs[:citeas][docid] and update_anchors(b, docxml, docid)
       end
     end
 
@@ -125,6 +144,9 @@ module Metanorma
     def update_indirect_refs_to_docs1(docxml, schema, id, file)
       docxml.xpath(ns("//eref[@bibitemid = '#{schema}_#{id}']")).each do |e|
         e["citeas"] = file
+        if a = e.at(ns(".//locality[@type = 'anchor']/referenceFrom"))
+          a.children = "#{a.text}_#{Metanorma::Utils::to_ncname(file)}"
+        end
       end
       docid = docxml.at(ns("//bibitem[@id = '#{schema}_#{id}']/"\
                            "docidentifier[@type = 'repository']")) or return
@@ -134,21 +156,19 @@ module Metanorma
 
     # update crossrefences to other documents, to include
     # disambiguating document suffix on id
-    def update_anchors(bib, docxml, _id) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      docid = bib&.at(ns("./docidentifier"))&.text
+    def update_anchors(bib, docxml, docid) # rubocop:disable Metrics/AbcSize
       docxml.xpath("//xmlns:eref[@citeas = '#{docid}']").each do |e|
-        if @files[docid]
-          update_anchor_loc(bib, e, docid)
+        if @files[docid] then update_anchor_loc(bib, e, docid)
         else
-          e << "<strong>** Unresolved reference to document #{docid}, "\
-            "id #{e['bibitemid']}</strong>"
+          e << "<strong>** Unresolved reference to document #{docid} "\
+            "from eref</strong>"
         end
       end
     end
 
-    def update_anchor_loc(bib, e, docid)
-      loc = e.at(ns(".//locality[@type = 'anchor']")) or
-        return update_anchor_create_loc(bib, e, docid)
+    def update_anchor_loc(bib, eref, docid)
+      loc = eref.at(ns(".//locality[@type = 'anchor']")) or
+        return update_anchor_create_loc(bib, eref, docid)
       document_suffix = Metanorma::Utils::to_ncname(docid)
       ref = loc.at(ns("./referenceFrom")) || return
       anchor = "#{ref.text}_#{document_suffix}"
@@ -161,17 +181,14 @@ module Metanorma
 
     # if there is a crossref to another document, with no anchor, retrieve the
     # anchor given the locality, and insert it into the crossref
-    def update_anchor_create_loc(bib, e, docid)
-      ins = e.at(ns("./localityStack")) || return
+    def update_anchor_create_loc(_bib, eref, docid)
+      ins = eref.at(ns("./localityStack")) or return
       type = ins&.at(ns("./locality/@type"))&.text
+      type = "clause" if type == "annex"
       ref = ins&.at(ns("./locality/referenceFrom"))&.text
-      (anchor = @files[docid][:anchors][type][ref]) || return
-      ref_from = Nokogiri::XML::Node.new "referenceFrom", bib
-      ref_from.content = anchor.sub(/^_/, "")
-      locality = Nokogiri::XML::Node.new "locality", bib
-      locality[:type] = "anchor"
-      locality.add_child ref_from
-      ins << locality
+      anchor = @files[docid][:anchors].dig(type, ref) or return
+      ins << "<locality type='anchor'><referenceFrom>#{anchor.sub(/^_/, '')}"\
+        "</referenceFrom></locality>"
     end
 
     # gather internal bibitem references
@@ -182,7 +199,7 @@ module Metanorma
         file, = targetfile(x, read: true)
         Nokogiri::XML(file)
           .xpath(ns("//bibitem[@type = 'internal']/"\
-        "docidentifier[@type = 'repository']")).each do |d|
+                    "docidentifier[@type = 'repository']")).each do |d|
           a = d.text.split(%r{/}, 2)
           a.size > 1 or next
           refs[a[0]] ||= {}
@@ -207,10 +224,11 @@ module Metanorma
 
     def locate_internal_refs1(refs, identifier, filedesc)
       file, _filename = targetfile(filedesc, read: true)
-      docxml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
+      t = xml.xpath("//*/@id").each_with_object({}) { |i, x| x[i.text] = true }
       refs.each do |schema, ids|
-        ids.each_key do |id|
-          n = docxml.at("//*[@id = '#{id}']") and
+        ids.keys.select { |id| t[id] }.each do |id|
+          n = xml.at("//*[@id = '#{id}']") and
             n.at("./ancestor-or-self::*[@type = '#{schema}']") and
             refs[schema][id] = identifier
         end

data/lib/metanorma/collection_fileprocess.rb

@@ -15,7 +15,7 @@ module Metanorma
     def read_files(path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       files = {}
       @xml.xpath(ns("//docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
+        identifier = d.at(ns("./identifier")).children.to_xml
         files[identifier] = file_entry(d, identifier, path)
         if files[identifier][:attachment]
           files[identifier][:bibdata] = Metanorma::Document
@@ -27,22 +27,57 @@ module Metanorma
           files[identifier][:anchors] = read_anchors(xml)
           files[identifier][:bibdata] = xml.at(ns("//bibdata"))
         end
+        files[identifier][:bibitem] = files[identifier][:bibdata].dup
+        files[identifier][:bibitem].name = "bibitem"
+        files[identifier][:bibitem]["hidden"] = "true"
+        files[identifier][:bibitem]&.at("./*[local-name() = 'ext']")&.remove
       end
-      files
+      add_section_split(files)
+    end
+
+    def add_section_split(files)
+      files.keys.each_with_object({}) do |k, m|
+        if files[k][:sectionsplit] == "true" && !files[k]["attachment"]
+          sectionsplit(files[k][:rel_path]).each_with_index do |f1, i|
+            m[k + f1[:title]] =
+              { parentid: k, presentationxml: true, type: "fileref",
+                rel_path: f1[:url], out_path: File.basename(f1[:url]),
+                anchors: read_anchors(Nokogiri::XML(File.read(f1[:url]))),
+                bibdata: files[k][:bibdata], ref: f1[:url] }
+            m[k + f1[:title]][:bare] = true unless i.zero?
+          end
+        end
+        m[k] = files[k]
+      end
+    end
+
+    def sectionsplit(file)
+      Compile.new.compile(
+        file, { format: :asciidoc, extension_keys: [:presentation] }
+        .merge(@compile_options)
+      )
+      r = file.sub(/\.xml$/, ".presentation.xml")
+      @isodoc.sectionsplit(
+        Nokogiri::XML(File.read(r)), File.basename(r), File.dirname(r)
+      ).sort_by { |f| f[:order] }
     end
 
     # rel_path is the source file address, determined relative to the YAML.
     # out_path is the destination file address, with any references outside
     # the working directory (../../...) truncated
-    def file_entry(docref, identifier, _path)
-      ret = if docref["fileref"]
+    def file_entry(ref, identifier, _path)
+      out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
+      ret = if ref["fileref"]
               { type: "fileref", ref: @documents[identifier].file,
-                rel_path: docref["fileref"],
-                out_path: Util::source2dest_filename(docref["fileref"]) }
+                rel_path: ref["fileref"],
+                out_path: out }
             else
-              { type: "id", ref: docref["id"] }
+              { type: "id", ref: ref["id"] }
             end
-      ret[:attachment] = docref["attachment"] if docref["attachment"]
+      ret[:attachment] = ref["attachment"] if ref["attachment"]
+      ret[:sectionsplit] = ref["sectionsplit"] if ref["sectionsplit"]
+      ret[:presentationxml] = ref["presentation-xml"] if ref["presentation-xml"]
+      ret[:bareafterfirst] = ref["bare-after-first"] if ref["bare-after-first"]
       ret
     end
 
@@ -74,9 +109,9 @@ module Metanorma
     # @return [Array<String, nil>]
     def targetfile(data, options)
       options = { read: false, doc: true, relative: false }.merge(options)
-      path = options[:relative] ? data[:out_path] : data[:ref]
+      path = options[:relative] ? data[:rel_path] : data[:ref]
       if data[:type] == "fileref"
-        ref_file path, options[:read], options[:doc]
+        ref_file path, data[:out_path], options[:read], options[:doc]
       else
         xml_file data[:id], options[:read]
       end
@@ -86,40 +121,71 @@ module Metanorma
     # @param read [Boolean]
     # @param doc [Boolean]
     # @return [Array<String, nil>]
-    def ref_file(ref, read, doc)
+    def ref_file(ref, out, read, doc)
       file = File.read(ref, encoding: "utf-8") if read
-      filename = ref.dup
+      filename = out.dup
       filename.sub!(/\.xml$/, ".html") if doc
       [file, filename]
     end
 
     # compile and output individual file in collection
+    # warn "metanorma compile -x html #{f.path}"
     def file_compile(file, filename, identifier)
-      # warn "metanorma compile -x html #{f.path}"
       c = Compile.new
-      c.compile file.path, { format: :asciidoc,
-                             extension_keys: @format }.merge(@compile_options)
+      c.compile file.path, { format: :asciidoc, extension_keys: @format }
+        .merge(compile_options(identifier))
       @files[identifier][:outputs] = {}
+      file_compile_formats(file, filename, identifier, c)
+    end
+
+    def compile_options(identifier)
+      ret = @compile_options.dup
+      Array(@directives).include?("presentation-xml") ||
+        @files[identifier][:presentationxml] and
+        ret.merge!(passthrough_presentation_xml: true)
+      @files[identifier][:sectionsplit] == "true" and
+        ret.merge!(sectionsplit: "true")
+      @files[identifier][:bare] == true and
+        ret.merge!(bare: true)
+      ret
+    end
+
+    def file_compile_formats(file, filename, identifier, compile)
       @format.each do |e|
-        ext = c.processor.output_formats[e]
-        fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
-        FileUtils.mv file.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
-        @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        ext = compile.processor.output_formats[e]
+        fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
+        if /html$/.match?(ext) && @files[identifier][:sectionsplit]
+          # file_sectionsplit_copy(file, fn, identifier, ext, e)
+        else
+          FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"),
+                       File.join(@outdir, fn)
+          @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        end
       end
     end
 
+    def file_sectionsplit_copy(file, base, identifier, ext, format)
+      dir = file.path.sub(/\.xml$/, ".#{ext}_collection")
+      files = Dir.glob("#{dir}/*.#{ext}")
+      FileUtils.cp files, @outdir
+      cover = File.join(@outdir, base.sub(/\.html$/, ".index.html"))
+      FileUtils.cp File.join(dir, "index.html"), cover
+      @files[identifier][:outputs][format] = cover
+    end
+
     def copy_file_to_dest(fileref)
-      _file, filename = targetfile(fileref, read: true, doc: false)
       dest = File.join(@outdir, fileref[:out_path])
       FileUtils.mkdir_p(File.dirname(dest))
-      FileUtils.cp filename, dest
+      FileUtils.cp fileref[:ref], dest
     end
 
     # process each file in the collection
     # files are held in memory, and altered as postprocessing
     def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       internal_refs = locate_internal_refs
-      @files.each do |identifier, x|
+      @files.each_with_index do |(identifier, x), i|
+        i.positive? && Array(@directives).include?("bare-after-first") and
+          @compile_options.merge!(bare: true)
         if x[:attachment] then copy_file_to_dest(x)
         else
           file, filename = targetfile(x, read: true)

data/lib/metanorma/collection_manifest.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "util"
+
 module Metanorma
   # Metanorma collection's manifest
   class CollectionManifest
@@ -15,6 +17,7 @@ module Metanorma
       @title = title
       @docref = docref
       @manifest = manifest
+      @disambig = Util::DisambigFiles.new
     end
 
     class << self
@@ -43,9 +46,11 @@ module Metanorma
       # @return [Hash{String=>String}]
       def parse_docref(mnf)
         mnf.xpath("xmlns:docref").map do |dr|
-          h = { "identifier" => dr.at("identifier").text }
+          h = { "identifier" => dr.at("identifier").children.to_xml }
           dr[:fileref] and h["fileref"] = dr[:fileref]
           h["attachment"] = dr[:attachment] if dr[:attachment]
+          h["sectionsplit"] = dr[:sectionsplit] if dr[:sectionsplit]
+          h["presentation-xml"] = dr[:presentationxml] if dr[:presentationxml]
           h
         end
       end
@@ -57,7 +62,7 @@ module Metanorma
       @manifest.each { |mnf| mnf.collection = col }
     end
 
-    # @param dir [String] path to coolection
+    # @param dir [String] path to collection
     # @return [Hash<String, Metanorma::Document>]
     def documents(dir = "")
       docs = @docref.each_with_object({}) do |dr, m|
@@ -100,14 +105,28 @@ module Metanorma
 
     # @param builder [Nokogiri::XML::Builder]
     def docref_to_xml(builder)
+      @disambig = Util::DisambigFiles.new
       @docref.each do |dr|
-        drf = builder.docref { |b| b.identifier dr["identifier"] }
-        drf[:fileref] = Util::source2dest_filename(dr["fileref"])
-        drf[:attachment] = dr["attachment"] if dr["attachment"]
-        if collection.directives.include?("documents-inline")
-          id = collection.documents.find_index { |k, _| k == dr["identifier"] }
-          drf[:id] = format("doc%<index>09d", index: id)
+        drf = builder.docref do |b|
+          b.identifier do |i|
+            i << dr["identifier"]
+          end
+        end
+        docref_to_xml_attrs(drf, dr)
+      end
+    end
+
+    def docref_to_xml_attrs(elem, docref)
+      elem[:fileref] = @disambig.source2dest_filename(docref["fileref"])
+      elem[:attachment] = docref["attachment"] if docref["attachment"]
+      elem[:sectionsplit] = docref["sectionsplit"] if docref["sectionsplit"]
+      elem[:presentationxml] = "true" if docref["presentation-xml"] &&
+        docref["presentation-xml"] == "true" || docref["presentation-xml"] == true
+      if collection.directives.include?("documents-inline")
+        id = collection.documents.find_index do |k, _|
+          k == docref["identifier"]
         end
+        elem[:id] = format("doc%<index>09d", index: id)
       end
     end
   end

data/lib/metanorma/collection_renderer.rb

@@ -3,6 +3,7 @@
 require "isodoc"
 require_relative "collection_fileprocess"
 require_relative "fontist_utils"
+require_relative "util"
 
 module Metanorma
   # XML collection renderer
@@ -34,13 +35,16 @@ module Metanorma
 
       @outdir = options[:output_folder]
       @coverpage = options[:coverpage]
-      @format = options[:format]
+      @format = Util.sort_extensions_execution(options[:format])
       @compile_options = options[:compile] || {}
       @log = options[:log]
       @documents = collection.documents
+      @directives = collection.directives
+      @disambig = Util::DisambigFiles.new
 
       # list of files in the collection
       @files = read_files folder
+      isodoc_populate(@isodoc)
       FileUtils.rm_rf @outdir
       FileUtils.mkdir_p @outdir
     end
@@ -66,13 +70,15 @@ module Metanorma
         ext = e == :presentation ? "presentation.xml" : e.to_s
         out = col.clone
         out.directives << "documents-inline"
-        out.documents.keys.each do |id|
+        out.documents.each_key do |id|
           next if @files[id][:attachment]
 
           filename = @files[id][:outputs][e]
           out.documents[id] = Metanorma::Document.raw_file(filename)
         end
-        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") { |f| f.write(out.to_xml) }
+        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") do |f|
+          f.write(out.to_xml)
+        end
       end
       options[:format].include?(:pdf) and
         pdfconv.convert(File.join(@outdir, "collection.presentation.xml"))
@@ -103,15 +109,21 @@ module Metanorma
     end
 
     # The isodoc class for the metanorma flavour we are using
-    def isodoc # rubocop:disable Metrics/MethodLength
+    def isodoc
       x = Asciidoctor.load nil, backend: @doctype.to_sym
       isodoc = x.converter.html_converter(Dummy.new)
       isodoc.i18n_init(@lang, @script) # read in internationalisation
+      isodoc.metadata_init(@lang, @script, isodoc.i18n)
+      isodoc.info(@xml, nil)
+      isodoc
+    end
+
+    def isodoc_populate(isodoc)
       # create the @meta class of isodoc, with "navigation" set to the index bar
       # extracted from the manifest
       nav = indexfile(@xml.at(ns("//manifest")))
       i18n = isodoc.i18n
-      i18n.set(:navigation, nav)
+      i18n.set("navigation", nav)
       isodoc.metadata_init(@lang, @script, i18n)
       # populate the @meta class of isodoc with the various metadata fields
       # native to the flavour; used to populate Liquid
@@ -168,11 +180,19 @@ module Metanorma
     # @param builder [Nokogiri::XML::Builder]
     def docrefs(elm, builder)
       elm.xpath(ns("./docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
-        link = if d["fileref"] then d["fileref"].sub(/\.xml$/, ".html")
-               else d["id"] + ".html"
-               end
-        builder.li { builder.a identifier, href: link }
+        ident = d.at(ns("./identifier")).children.to_xml
+        builder.li do |li|
+          li.a **{ href: index_link(d, ident) } do |a|
+            a << ident
+          end
+        end
+      end
+    end
+
+    def index_link(docref, ident)
+      if docref["fileref"]
+        @files[ident][:out_path].sub(/\.xml$/, ".html")
+      else "#{docref['id']}.html"
       end
     end
 
@@ -198,6 +218,14 @@ module Metanorma
 
     private
 
+    def format_sort(formats)
+      ret = []
+      formats.include?(:xml) and ret << :xml
+      formats.include?(:presentation) and ret << :presentation
+      a = %i(presentation xml)
+      ret + formats.reject { |i| a.include? i }
+    end
+
     # @param options [Hash]
     # @raise [ArgumentError]
     def check_options(options)

data/lib/metanorma/compile.rb

@@ -6,6 +6,7 @@ require "fontist"
 require "fontist/manifest/install"
 require_relative "compile_validate"
 require_relative "fontist_utils"
+require_relative "util"
 
 module Metanorma
   class Compile
@@ -35,7 +36,7 @@ module Metanorma
     end
 
     def xml_options_extract(file)
-      xml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
       if xml.root
         @registry.root_tags.each do |k, v|
           return { type: k } if v == xml.root.name
@@ -74,7 +75,9 @@ module Metanorma
           memo
         end
       end
-      if !extensions.include?(:presentation) and extensions.any? { |e| @processor.use_presentation_xml(e) }
+      if !extensions.include?(:presentation) && extensions.any? do |e|
+        @processor.use_presentation_xml(e)
+      end
         extensions << :presentation
       end
       extensions
@@ -88,7 +91,7 @@ module Metanorma
         options[:asciimath] and
           file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
         dir = File.dirname(filename)
-        dir != '.' and
+        dir != "." and
           file.gsub!(/^include::/, "include::#{dir}/")
         [file, @processor.input_to_isodoc(file, filename, options)]
       when ".xml"
@@ -108,15 +111,17 @@ module Metanorma
 
     def relaton_export(isodoc, options)
       return unless options[:relaton]
-      xml = Nokogiri::XML(isodoc)
+
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
-      #docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
-      #outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
+      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
+      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
       File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
     end
 
     def clean_sourcecode(xml)
-      xml.xpath(".//callout | .//annotation | .//xmlns:callout | .//xmlns:annotation").each do |x|
+      xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
+                ".//xmlns:annotation").each do |x|
         x.remove
       end
       xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
@@ -125,12 +130,13 @@ module Metanorma
 
     def extract(isodoc, dirname, extract_types)
       return unless dirname
+
       if extract_types.nil? || extract_types.empty?
         extract_types = [:sourcecode, :image, :requirement]
       end
       FileUtils.rm_rf dirname
       FileUtils.mkdir_p dirname
-      xml = Nokogiri::XML(isodoc)
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
       image_export(xml, dirname) if extract_types.include? :image
       requirement_export(xml, dirname) if extract_types.include? :requirement
@@ -176,17 +182,6 @@ module Metanorma
       end
     end
 
-    # dependency ordering
-    def sort_extensions_execution(ext)
-      case ext
-      when :xml then 0
-      when :rxl then 1
-      when :presentation then 2
-      else
-        99
-      end
-    end
-
     def wrap_html(options, file_extension, outfilename)
       if options[:wrapper] && /html$/.match(file_extension)
         outfilename = outfilename.sub(/\.html$/, "")
@@ -201,34 +196,22 @@ module Metanorma
       f = change_output_dir options
       xml_name = f.sub(/\.[^.]+$/, ".xml")
       presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
-      extensions.sort do |a, b|
-        sort_extensions_execution(a) <=> sort_extensions_execution(b)
-      end.each do |ext|
-        isodoc_options = @processor.extract_options(file)
-        isodoc_options[:datauriimage] = true if options[:datauriimage]
-        isodoc_options[:sourcefilename] = options[:filename]
+      Util.sort_extensions_execution(extensions).each do |ext|
         file_extension = @processor.output_formats[ext]
         outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        if ext == :pdf
-          font_locations = FontistUtils.fontist_font_locations(@processor, options)
-          font_locations and
-            isodoc_options[:mn2pdf] = { font_manifest_file: font_locations.path }
-        end
+        isodoc_options = get_isodoc_options(file, options, ext)
         if ext == :rxl
           options[:relaton] = outfilename
           relaton_export(isodoc, options)
+        elsif options[:passthrough_presentation_xml] && ext == :presentation
+          FileUtils.cp f, presentationxml_name
         else
           begin
             @processor.use_presentation_xml(ext) ?
               @processor.output(nil, presentationxml_name, outfilename, ext, isodoc_options) :
               @processor.output(isodoc, xml_name, outfilename, ext, isodoc_options)
           rescue StandardError => e
-            if e.message.include? "Fatal:"
-              @errors << e.message
-            else
-              puts e.message
-              puts e.backtrace.join("\n")
-            end
+            isodoc_error_process(e)
           end
         end
         wrap_html(options, file_extension, outfilename)
@@ -237,6 +220,28 @@ module Metanorma
 
     private
 
+    def isodoc_error_process(err)
+      if err.message.include? "Fatal:"
+        @errors << err.message
+      else
+        puts err.message
+        puts err.backtrace.join("\n")
+      end
+    end
+
+    def get_isodoc_options(file, options, ext)
+      isodoc_options = @processor.extract_options(file)
+      isodoc_options[:datauriimage] = true if options[:datauriimage]
+      isodoc_options[:sourcefilename] = options[:filename]
+      isodoc_options[:bare] ||= options[:bare]
+      isodoc_options[:sectionsplit] ||= options[:sectionsplit]
+      if ext == :pdf
+        floc = FontistUtils.fontist_font_locations(@processor, options) and
+          isodoc_options[:mn2pdf] = { font_manifest_file: floc.path }
+      end
+      isodoc_options
+    end
+
     # @param options [Hash]
     # @return [String]
     def change_output_dir(options)

data/lib/metanorma/document.rb

@@ -30,7 +30,9 @@ module Metanorma
 
       # raw XML file, can be used to put in entire file instead of just bibitem
       def raw_file(filename)
-        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8"))
+        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8")) do |config|
+          config.huge
+        end
         new(doc, filename, raw: true)
       end
 
@@ -100,7 +102,9 @@ module Metanorma
       if @raw
         builder << @bibitem.root.to_xml
       else
-        builder.send(type + "-standard") { |b| b << @bibitem.to_xml(bibdata: true) }
+        builder.send("#{type}-standard") do |b|
+          b << @bibitem.to_xml(bibdata: true)
+        end
       end
     end
   end

data/lib/metanorma/input/asciidoc.rb

@@ -2,20 +2,14 @@ require "nokogiri"
 
 module Metanorma
   module Input
-
     class Asciidoc < Base
-
       def process(file, filename, type, options = {})
         require "asciidoctor"
         out_opts = {
-          to_file: false,
-          safe: :safe,
-          backend: type,
-          header_footer: true,
-          attributes: [
-            "nodoc", "stem", "xrefstyle=short", "docfile=#{filename}",
-            "output_dir=#{options[:output_dir]}"
-          ]
+          to_file: false, safe: :safe, backend: type, header_footer: true,
+          attributes: ["nodoc", "stem", "xrefstyle=short",
+                       "docfile=#{filename}",
+                       "output_dir=#{options[:output_dir]}"]
         }
         unless asciidoctor_validate(file, filename, out_opts)
           warn "Cannot continue compiling Asciidoctor document"
@@ -27,11 +21,12 @@ module Metanorma
       def asciidoctor_validate(file, filename, options)
         err = nil
         begin
-          previous_stderr, $stderr = $stderr, StringIO.new
+          previous_stderr = $stderr
+          $stderr = StringIO.new
           ::Asciidoctor.load(file, options)
-          %r{(\n|^)asciidoctor: ERROR: ['"]?#{Regexp.escape(filename || 
-            "<empty>")}['"]?: line \d+: include file not found: }.match($stderr.string) and
-              err = $stderr.string
+          %r{(\n|^)asciidoctor: ERROR: ['"]?#{Regexp.escape(filename ||
+            "<empty>")}['"]?: line \d+: include file not found: }
+            .match($stderr.string) and err = $stderr.string
         ensure
           $stderr = previous_stderr
         end
@@ -45,8 +40,9 @@ module Metanorma
         /\n:mn-output-extensions: (?<extensions>[^\n]+)\n/ =~ headerextract
         /\n:mn-relaton-output-file: (?<relaton>[^\n]+)\n/ =~ headerextract
         /\n(?<asciimath>:mn-keep-asciimath:[^\n]*)\n/ =~ headerextract
-        asciimath = defined?(asciimath) ?
-          (!asciimath.nil? && asciimath != ":mn-keep-asciimath: false") : nil
+        asciimath = if defined?(asciimath)
+                      (!asciimath.nil? && asciimath != ":mn-keep-asciimath: false")
+                    end
         asciimath = nil if asciimath == false
         {
           type: defined?(type) ? type : nil,
@@ -60,38 +56,39 @@ module Metanorma
         attr&.sub(/^#{name}:\s*$/, "#{name}: true")&.sub(/^#{name}:\s+/, "")
       end
 
+      ADOC_OPTIONS = %w(htmlstylesheet htmlcoverpage htmlintropage scripts
+                        scripts-override scripts-pdf wordstylesheet i18nyaml
+                        standardstylesheet header wordcoverpage wordintropage
+                        ulstyle olstyle htmlstylesheet-override bare
+                        htmltoclevels doctoclevels sectionsplit
+                        body-font header-font monospace-font title-font
+                        wordstylesheet-override).freeze
+
       def extract_options(file)
         header = file.sub(/\n\n.*$/m, "\n")
-        ret = %w(htmlstylesheet htmlcoverpage htmlintropage scripts
-                 scripts-pdf wordstylesheet
-                 standardstylesheet header wordcoverpage wordintropage i18nyaml
-                 ulstyle olstyle htmlstylesheet-override
-                 htmltoclevels doctoclevels sectionsplit
-                 body-font header-font monospace-font title-font
-                 wordstylesheet-override).each_with_object({}) do |w, acc|
+        ret = ADOC_OPTIONS.each_with_object({}) do |w, acc|
           m = /\n:#{w}: ([^\n]+)\n/.match(header) or next
           acc[w.gsub(/-/, "").sub(/override$/, "_override")
             .sub(/pdf$/, "_pdf").to_sym] = m[1]
         end
         /\n:data-uri-image: (?<datauriimage>[^\n]+)\n/ =~ header
-        /\n:(?<hierarchical_assets>hierarchical-assets:[^\n]*)\n/ =~ header
+        /\n:(?<hier_assets>hierarchical-assets:[^\n]*)\n/ =~ header
         /\n:(?<use_xinclude>use-xinclude:[^\n]*)\n/ =~ header
         /\n:(?<break_up>break-up-urls-in-tables:[^\n]*)\n/ =~ header
 
-        defined?(hierarchical_assets) and
-          hierarchical_assets = empty_attr(hierarchical_assets, "hierarchical-assets")
+        defined?(hier_assets) and
+          hier_assets = empty_attr(hier_assets, "hierarchical-assets")
         defined?(use_xinclude) and
           use_xinclude = empty_attr(use_xinclude, "use-xinclude")
         defined?(break_up) and
           break_up = empty_attr(break_up, "break-up-urls-in-tables")
-        ret.merge({
+        ret.merge(
           datauriimage: defined?(datauriimage) ? datauriimage != "false" : nil,
-          hierarchical_assets: defined?(hierarchical_assets) ? hierarchical_assets : nil,
+          hierarchical_assets: defined?(hier_assets) ? hier_assets : nil,
           use_xinclude: defined?(use_xinclude) ? use_xinclude : nil,
           break_up_urls_in_tables: defined?(break_up) ? break_up : nil,
-        }).reject { |_, val| val.nil? }
+        ).reject { |_, val| val.nil? }
       end
-
     end
   end
 end

data/lib/metanorma/util.rb

@@ -12,8 +12,49 @@ module Metanorma
       end
     end
 
-    def self.source2dest_filename(name)
-      name.sub(%r{^(\./)?(\.\./)+}, "")
+    # dependency ordering
+    def self.sort_extensions_execution_ord(ext)
+      case ext
+      when :xml then 0
+      when :rxl then 1
+      when :presentation then 2
+      else
+        99
+      end
+    end
+
+    def self.sort_extensions_execution(ext)
+      ext.sort do |a, b|
+        sort_extensions_execution_ord(a) <=> sort_extensions_execution_ord(b)
+      end
+    end
+
+    class DisambigFiles
+      def initialize
+        @seen_filenames = []
+      end
+
+      def source2dest_filename(name, disambig = true)
+        n = name.sub(%r{^(\./)?(\.\./)+}, "")
+        dir = File.dirname(n)
+        base = File.basename(n)
+        if disambig && @seen_filenames.include?(base)
+          base = disambiguate_filename(base)
+        end
+        @seen_filenames << base
+        dir == "." ? base : File.join(dir, base)
+      end
+
+      def disambiguate_filename(base)
+        m = /^(?<start>.+\.)(?!0)(?<num>\d+)\.(?<suff>[^.]*)$/.match(base) ||
+          /^(?<start>.+\.)(?<suff>[^.]*)/.match(base) ||
+          /^(?<start>.+)$/.match(base)
+        i = m.names.include?("num") ? m["num"].to_i + 1 : 1
+        while @seen_filenames.include? base = "#{m['start']}#{i}.#{m['suff']}"
+          i += 1
+        end
+        base
+      end
     end
   end
 end

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.3.3".freeze
+  VERSION = "1.3.7".freeze
 end

data/metanorma.gemspec

@@ -20,10 +20,10 @@ Gem::Specification.new do |spec|
   spec.bindir        = "bin"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.4.0"
+  spec.required_ruby_version = ">= 2.5.0"
 
   spec.add_runtime_dependency "asciidoctor"
-  spec.add_runtime_dependency "fontist", "~> 1.8"
+  spec.add_runtime_dependency "fontist", "~> 1.9"
   spec.add_runtime_dependency "htmlentities"
   spec.add_runtime_dependency "metanorma-utils", "~> 1.2.0"
   spec.add_runtime_dependency "mn2pdf", "~> 1"
@@ -36,10 +36,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "byebug", "~> 10.0"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "metanorma-iso", "~> 1.8.0"
+  spec.add_development_dependency "metanorma-iso", "~> 1.9.0"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-command", "~> 1.0"
   spec.add_development_dependency "rubocop", "~> 1.5.2"
   spec.add_development_dependency "sassc", "~> 2.4.0"
+  spec.add_development_dependency "mnconvert"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.7
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-24 00:00:00.000000000 Z
+date: 2021-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +220,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.4.0
+- !ruby/object:Gem::Dependency
+  name: mnconvert
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Library to process any Metanorma standard.
 email:
 - open.source@ribose.com
@@ -286,7 +300,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="