metanorma 1.6.4 → 1.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abbc191b2e898bbeeea46f7867bb9d25d44cfd7f9a0dd2c690fcbdb8383136fd
-  data.tar.gz: a9e184662fcf25a575c278d604440b8215fd7b54b76f980a4cf4bfd5f8a1f543
+  metadata.gz: c329757741717214253fd1ca5cf31cfaa1859c2c35647b5586970cb36595f1a3
+  data.tar.gz: c02920016d7294a37d9459e26d9d4e5c4291bc75a94181c1d139ad7f3a2e5215
 SHA512:
-  metadata.gz: bfc28efd7f7f48f0bfd7dfa630bf88f1186006022dc6d5f82f188a3749304bc73837c9e96dd9246655de67f2b1ee29d9525f49c2ba568e2116d4329bf3689bda
-  data.tar.gz: 40bc185f4dbe9b4570349e22a15334a2e3fb089dfe3002ab43cb4a8ffa4afe4f2a3ae8357f80e61d585a57ab3b64e24dd5fb2700241dd6f1b958693214afdbc6
+  metadata.gz: d8fb74a61e67a308fb0523e4512bc4d2e0067d5dadb8cc736114e47781e9b638066a0e438cadeca16a1d210f785eeb544abd526a91483dfd81f6580b9fa051ff
+  data.tar.gz: de5e33cc1cfd207c6ad912759a5fc9e8cda3f576eaabd3e2c897662d299d298d37d9db052589990f5aa9f18a9ccd675d870212f0a07b987128d42eec198e5007

data/lib/metanorma/collection_fileparse.rb

@@ -1,8 +1,6 @@
 module Metanorma
   # XML collection renderer
   class CollectionRenderer
-    # @param bib [Nokogiri::XML::Element]
-    # @param identifier [String]
     def update_bibitem(bib, identifier)
       docid = get_bibitem_docid(bib, identifier) or return
       newbib = dup_bibitem(docid, bib)
@@ -14,77 +12,90 @@ module Metanorma
       bib.replace(newbib)
     end
 
-    def get_bibitem_docid(bib, identifier)
-      # IDs for repo references are untyped by default
-      docid = bib.at(ns("./docidentifier[not(@type)]")) ||
-        bib.at(ns("./docidentifier"))
-      docid &&= docid_prefix(docid)
-      if @files.get(docid) then docid
-      else
-        fail_update_bibitem(docid, identifier)
-        nil
-      end
-    end
-
-    def docid_prefix(docid)
-      type = docid["type"]
-      type == "metanorma-collection" and type = nil
-      @c.decode(@isodoc
-          .docid_prefix(type, docid.children.to_xml)).gsub(/\s/, " ")
-    end
-
-    def dup_bibitem(docid, bib)
-      newbib = @files.get(docid, :bibdata).dup
-      newbib.name = "bibitem"
-      newbib["hidden"] = "true"
-      newbib&.at("./*[local-name() = 'ext']")&.remove
-      newbib["id"] = bib["id"]
-      newbib
-    end
-
-    # Resolves direct links to other files in collection
-    # (repo(current-metanorma-collection/x),
-    # and indirect links to other files in collection
-    # (bibitem[@type = 'internal'] pointing to a file anchor
-    # in another file in the collection)
+    # Resolves references to other files in the collection. Three routines:
+    # 1. Eref to a document that has been split into multiple documents
+    # (sectionsplit) are resolved to direct eref to the split document
+    # 2. Indirect erefs to a file anchor in an unknwon file in the collection
+    # (bibitem[@type = 'internal'] ) are resolved to direct eref to the
+    # containing document
+    # 3. Direct erefs to other files in collection
+    # (repo(current-metanorma-collection/x) are resolved to hyperlinks
     # @param file [String] XML content
     # @param identifier [String] docid
     # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to
     #   anchor to filename
     # @return [String] XML content
-    def update_xrefs(file, identifier, internal_refs)
+    def update_xrefs(file, docid, internal_refs)
       docxml = file.is_a?(String) ? Nokogiri::XML(file, &:huge) : file
       supply_repo_ids(docxml)
-      update_indirect_refs_to_docs(docxml, internal_refs)
-      ids = @files.get(identifier, :ids)
-      @files.add_document_suffix(identifier, docxml)
-      update_direct_refs_to_docs(docxml, identifier)
+      @nested or update_indirect_refs_to_docs(docxml, docid, internal_refs)
+      @files.add_document_suffix(docid, docxml)
+      @nested or update_sectionsplit_refs_to_docs(docxml, internal_refs)
+      update_direct_refs_to_docs(docxml, docid)
       hide_refs(docxml)
-      @files.get(identifier, :sectionsplit_output) and eref2link(docxml)
-      svgmap_resolve(datauri_encode(docxml), ids)
+      @files.get(docid, :sectionsplit_output) and eref2link(docxml)
+      svgmap_resolve(datauri_encode(docxml), @files.get(docid, :ids))
       docxml.to_xml
     end
 
+    def update_sectionsplit_refs_to_docs(docxml, internal_refs)
+      Util::gather_citeases(docxml).each do |k, v|
+        (@files.get(k) && @files.get(k, :sectionsplit)) or next
+        opts = { key: @files.get(k, :indirect_key),
+                 source_suffix: docxml.root["document_suffix"],
+                 target_suffix: @files.get(k, :document_suffix) }
+        refs = v.each_with_object({}) do |eref, m|
+          update_sectionsplit_eref_to_doc(eref, internal_refs, m, opts)
+        end
+        add_hidden_bibliography(docxml, refs)
+      end
+    end
+
+    def update_sectionsplit_eref_to_doc(eref, internal_refs, doclist, opts)
+      a = eref.at(ns("./localityStack/locality[@type = 'anchor']/" \
+                     "referenceFrom")) or return
+      doc = internal_refs[opts[:key]]["#{a.text}_#{opts[:target_suffix]}"]
+      bibitemid = Metanorma::Utils::to_ncname("#{doc}_#{opts[:source_suffix]}")
+      eref["bibitemid"] = bibitemid
+      doclist[bibitemid] ||= doc
+      doclist
+    end
+
+    def new_hidden_ref(xmldoc)
+      ins = xmldoc.at(ns("bibliography")) or
+        xmldoc.root << "<bibliography/>" and ins = xmldoc.at(ns("bibliography"))
+      ins.at(ns("./referenced[@hidden = 'true']")) or
+        ins.add_child("<references hidden='true' normative='false'/>").first
+    end
+
+    def add_hidden_bibliography(xmldoc, refs)
+      ins = new_hidden_ref(xmldoc)
+      refs.each do |k, v|
+        _, url = @files.targetfile_id(v, {})
+        ins << <<~XML
+          <bibitem id="#{k}">
+            <docidentifier type="repository">current-metanorma-collection/#{v}</docidentifier>
+            <uri type='citation'>#{url}</uri>
+          </bibitem>
+        XML
+      end
+    end
+
     def eref2link(docxml)
       isodoc = IsoDoc::PresentationXMLConvert.new({})
       isodoc.bibitem_lookup(docxml)
       isodoc.eref2link(docxml)
     end
 
-    def hide_refs(docxml)
-      docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or " \
-                      "@hidden = 'false'])]")).each do |f|
-        f["hidden"] = "true"
-      end
-    end
-
-    def supply_repo_ids(docxml)
-      docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
-        b.at(ns("./docidentifier[@type = 'repository']")) and next
+    def supply_repo_ids(doc)
+      doc.xpath(
+        ns("//bibitem[not(ancestor::bibitem)]" \
+           "[not(./docidentifier[@type = 'repository'])]"),
+      ).each do |b|
         b.xpath(ns("./docidentifier")).each do |docid|
-          id = @isodoc
-            .docid_prefix(docid["type"], docid.children.to_xml)
+          id = @isodoc.docid_prefix(docid["type"], docid.children.to_xml)
           @files.get(id) or next
+          @files.get(id, :indirect_key) and next # will resolve as indirect key
           docid.next = "<docidentifier type='repository'>" \
                        "current-metanorma-collection/#{id}</docidentifier>"
         end
@@ -102,13 +113,12 @@ module Metanorma
     end
 
     def svgmap_resolve1(eref, isodoc, _docxml, ids)
-      href = isodoc.eref_target(eref)
-      return if href == "##{eref['bibitemid']}" ||
-        (href =~ /^#/ && !ids[href.sub(/^#/, "")])
-
+      href = isodoc.eref_target(eref) or return
+      href == "##{eref['bibitemid']}" ||
+        (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
       eref["target"] = href.strip
       eref.name = "link"
-      eref&.elements&.remove
+      eref.elements&.remove
     end
 
     # repo(current-metanorma-collection/ISO 17301-1:2016)
@@ -117,51 +127,83 @@ module Metanorma
     # Preferably with anchor, and is a job to realise dynamic lookup
     # of localities.
     def update_direct_refs_to_docs(docxml, identifier)
-      @ncnames = {}
-      erefs = Util::gather_citeases(docxml)
+      erefs, erefs1 = update_direct_refs_to_docs_prep(docxml)
       docxml.xpath(ns("//bibitem")).each do |b|
-        docid = b.at(ns("./docidentifier[@type = 'repository']"))
-        (docid && %r{^current-metanorma-collection/}.match(docid.text)) or next
+        docid = b.at(ns("./docidentifier[@type = 'repository']")) or next
+        strip_unresolved_repo_erefs(identifier, docid, erefs1, b) or next
         update_bibitem(b, identifier)
         docid = docid_to_citeas(b) or next
         erefs[docid] and update_anchors(b, docid, erefs[docid])
       end
     end
 
-    def docid_to_citeas(bib)
-      docid = bib.at(ns("./docidentifier[@primary = 'true']")) ||
-        bib.at(ns("./docidentifier")) or return
-      docid_prefix(docid)
+    def update_direct_refs_to_docs_prep(docxml)
+      @ncnames = {}
+      [Util::gather_citeases(docxml), Util::gather_bibitemids(docxml)]
     end
 
-    def collect_erefs(docxml)
-      docxml.xpath(ns("//eref"))
-        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
-        m[:citeas][i["citeas"]] = true
-        m[:bibitemid][i["bibitemid"]] = true
-      end
+    # strip erefs if they are repository erefs, but do not point to a document
+    # within the current collection. This can happen if a collection consists
+    # of many documents, but not all are included in the current collection.
+    # Do not do this if this is a sectionsplit collection or a nested manifest.
+    # Return false if bibitem is not to be further processed
+    def strip_unresolved_repo_erefs(_document_id, bib_docid, erefs, bibitem)
+      %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text) and
+        return true
+      @nested and return false
+      erefs[bibitem["id"]]&.each { |x| x.parent and strip_eref(x) }
+      false
     end
 
     # Resolve erefs to a container of ids in another doc,
     # to an anchor eref (direct link)
-    def update_indirect_refs_to_docs(docxml, internal_refs)
-      bibitems = Util::gather_bibitems(docxml)
-      erefs = Util::gather_bibitemids(docxml)
+    def update_indirect_refs_to_docs(docxml, _docidentifier, internal_refs)
+      bibitems, erefs = update_indirect_refs_to_docs_prep(docxml)
       internal_refs.each do |schema, ids|
         ids.each do |id, file|
-          update_indirect_refs_to_docs1(docxml, "#{schema}_#{id}",
+          k = indirect_ref_key(schema, id, docxml)
+          update_indirect_refs_to_docs1(docxml, k,
                                         file, bibitems, erefs)
         end
       end
     end
 
+    def update_indirect_refs_to_docs_prep(docxml)
+      bibitems = Util::gather_bibitems(docxml)
+      erefs = Util::gather_bibitemids(docxml)
+      @updated_anchors = {}
+      [bibitems, erefs]
+    end
+
+    def indirect_ref_key(schema, id, docxml)
+      /^#{schema}_/.match?(id) and return id
+      ret = "#{schema}_#{id}"
+      (k = docxml.root["type"]) && k != schema and
+        ret = "#{ret}_#{docxml.root['document_suffix']}"
+      ret
+    end
+
     def update_indirect_refs_to_docs1(_docxml, key, file, bibitems, erefs)
       erefs[key]&.each do |e|
         e["citeas"] = file
-        a = e.at(ns(".//locality[@type = 'anchor']/referenceFrom")) and
-          a.children = "#{a.text}_#{Metanorma::Utils::to_ncname(file)}"
+        update_indirect_refs_to_docs_anchor(e, file)
       end
-      docid = bibitems[key]&.at(ns("./docidentifier[@type = 'repository']")) or
+      update_indirect_refs_to_docs_docid(bibitems[key], file)
+    end
+
+    def update_indirect_refs_to_docs_anchor(eref, file)
+      a = eref.at(ns(".//locality[@type = 'anchor']/referenceFrom")) or return
+      suffix = file
+      @files.get(file) && p = @files.get(file, :parentid) and
+        suffix = "#{p}_#{suffix}"
+      existing = a.text
+      anchor = Metanorma::Utils::to_ncname("#{existing}_#{suffix}")
+      @updated_anchors[existing] or a.children = anchor
+      @updated_anchors[anchor] = true
+    end
+
+    def update_indirect_refs_to_docs_docid(bibitem, file)
+      docid = bibitem&.at(ns("./docidentifier[@type = 'repository']")) or
         return
       docid.children = "current-metanorma-collection/#{file}"
       docid.previous =
@@ -174,8 +216,11 @@ module Metanorma
       erefs.each do |e|
         if @files.get(docid) then update_anchor_loc(bib, e, docid)
         else
-          e << "<strong>** Unresolved reference to document #{docid} " \
-               "from eref</strong>"
+          msg = "<strong>** Unresolved reference to document #{docid} " \
+                "from eref</strong>"
+          e << msg
+          strip_eref(e)
+          @log&.add("Cross-References", e, msg)
         end
       end
     end
@@ -186,10 +231,9 @@ module Metanorma
       @ncnames[docid] ||= Metanorma::Utils::to_ncname(docid)
       ref = loc.at("./xmlns:referenceFrom") or return
       anchor = "#{ref.text}_#{@ncnames[docid]}"
-      return unless @files.get(docid, :anchors).inject([]) do |m, (_, x)|
+      @files.get(docid, :anchors).inject([]) do |m, (_, x)|
         m += x.values
-      end.include?(anchor)
-
+      end.include?(anchor) or return
       ref.content = anchor
     end
 
@@ -204,55 +248,5 @@ module Metanorma
       ins << "<locality type='anchor'><referenceFrom>#{anchor.sub(/^_/, '')}" \
              "</referenceFrom></locality>"
     end
-
-    # gather internal bibitem references
-    def gather_internal_refs
-      @files.keys.each_with_object({}) do |i, refs|
-        @files.get(i, :attachment) and next
-        file, = @files.targetfile_id(i, read: true)
-        gather_internal_refs1(file, refs)
-      end
-    end
-
-    def gather_internal_refs1(file, refs)
-      Nokogiri::XML(file, &:huge)
-        .xpath(ns("//bibitem[@type = 'internal']/" \
-                  "docidentifier[@type = 'repository']")).each do |d|
-        a = d.text.split(%r{/}, 2)
-        a.size > 1 or next
-        refs[a[0]] ||= {}
-        refs[a[0]][a[1]] = true
-      end
-    end
-
-    # resolve file location for the target of each internal reference
-    def locate_internal_refs
-      refs = gather_internal_refs
-      @files.keys.reject { |k| @files.get(k, :attachment) }.each do |ident|
-        locate_internal_refs1(refs, ident, @isodoc.docid_prefix("", ident.dup))
-      end
-      refs.each do |schema, ids|
-        ids.each do |id, key|
-          key == true and refs[schema][id] = "Missing:#{schema}:#{id}"
-        end
-      end
-      refs
-    end
-
-    def locate_internal_refs1(refs, identifier, ident)
-      t = locate_internal_refs1_prep(ident)
-      refs.each do |schema, ids|
-        ids.keys.select { |id| t[id] }.each do |id|
-          t[id].at("./ancestor-or-self::*[@type = '#{schema}']") and
-            refs[schema][id] = identifier
-        end
-      end
-    end
-
-    def locate_internal_refs1_prep(ident)
-      file, _filename = @files.targetfile_id(ident, read: true)
-      xml = Nokogiri::XML(file, &:huge)
-      xml.xpath("//*[@id]").each_with_object({}) { |i, x| x[i["id"]] = i }
-    end
   end
 end

data/lib/metanorma/collection_fileprocess.rb

@@ -75,5 +75,82 @@ module Metanorma
         end
       end
     end
+
+    # gather internal bibitem references
+    def gather_internal_refs
+      @files.keys.each_with_object({}) do |i, refs|
+        @files.get(i, :attachment) and next
+        file, = @files.targetfile_id(i, read: true)
+        gather_internal_refs1(file, i, refs)
+      end
+    end
+
+    def gather_internal_refs1(file, ident, refs)
+      f = Nokogiri::XML(file, &:huge)
+      !@files.get(ident, :sectionsplit) and
+        gather_internal_refs_indirect(f, refs)
+      key = @files.get(ident, :indirect_key) and
+        gather_internal_refs_sectionsplit(f, ident, key, refs)
+    end
+
+    def gather_internal_refs_indirect(doc, refs)
+      doc.xpath(ns("//bibitem[@type = 'internal']/" \
+                   "docidentifier[@type = 'repository']")).each do |d|
+                     a = d.text.split(%r{/}, 2)
+                     a.size > 1 or next
+                     refs[a[0]] ||= {}
+                     refs[a[0]][a[1]] = false
+                   end
+    end
+
+    def gather_internal_refs_sectionsplit(_doc, ident, key, refs)
+      refs[key] ||= {}
+      @files.get(ident, :ids).each_key do |k|
+        refs[key][k] = false
+      end
+    end
+
+    def populate_internal_refs(refs)
+      @files.keys.reject do |k|
+        @files.get(k, :attachment) || @files.get(k, :sectionsplit)
+      end.each do |ident|
+        locate_internal_refs1(refs, ident, @isodoc.docid_prefix("", ident.dup))
+      end
+      refs
+    end
+
+    # resolve file location for the target of each internal reference
+    def locate_internal_refs
+      refs = populate_internal_refs(gather_internal_refs)
+      refs.each do |schema, ids|
+        ids.each do |id, key|
+          key and next
+          refs[schema][id] = "Missing:#{schema}:#{id}"
+          @log&.add("Cross-References", nil, refs[schema][id])
+        end
+      end
+      refs
+    end
+
+    def locate_internal_refs1(refs, identifier, ident)
+      t = locate_internal_refs1_prep(ident)
+      refs.each do |schema, ids|
+        ids.keys.select { |id| t[id] }.each do |id|
+          t[id].at("./ancestor-or-self::*[@type = '#{schema}']") and
+            refs[schema][id] = identifier
+        end
+      end
+    end
+
+    def locate_internal_refs1_prep(ident)
+      file, = @files.targetfile_id(ident, read: true)
+      xml = Nokogiri::XML(file, &:huge)
+      r = xml.root["document_suffix"]
+      xml.xpath("//*[@id]").each_with_object({}) do |i, x|
+        /^semantic_/.match?(i.name) and next
+        x[i["id"]] = i
+        r and x[i["id"].sub(/_#{r}$/, "")] = i
+      end
+    end
   end
 end

data/lib/metanorma/collection_manifest.rb

@@ -66,10 +66,9 @@ module Metanorma
     # @return [Hash<String, Metanorma::Document>]
     def documents(dir = "")
       docs = @docref.each_with_object({}) do |dr, m|
-        next m unless dr["fileref"]
-
+        dr["fileref"] or next m
         m[dr["identifier"]] = Document.parse_file(
-          File.join(dir, dr["fileref"]),
+          rel_path_resolve(dir, dr["fileref"]),
           dr["attachment"], dr["identifier"], dr["index"]
         )
         m
@@ -77,6 +76,11 @@ module Metanorma
       @manifest.reduce(docs) { |mem, mnf| mem.merge mnf.documents(dir) }
     end
 
+    def rel_path_resolve(dir, path)
+      p = Pathname.new(path)
+      p.absolute? ? path : File.join(dir, path)
+    end
+
     # @param builder [Nokogiri::XML::Builder]
     def to_xml(builder)
       builder.manifest do |b|
@@ -118,7 +122,7 @@ module Metanorma
     end
 
     def docref_to_xml_attrs(elem, docref)
-      elem[:fileref] = @disambig.source2dest_filename(docref["fileref"])
+      elem[:fileref] = @disambig.strip_root(docref["fileref"])
       %i(attachment sectionsplit).each do |i|
         elem[i] = docref[i.to_s] if docref[i.to_s]
       end

data/lib/metanorma/collection_render_utils.rb

@@ -10,8 +10,62 @@ module Metanorma
                 path.basename.to_s.gsub(clean_regex, fallback_sym))
     end
 
+    def dup_bibitem(docid, bib)
+      newbib = @files.get(docid, :bibdata).dup
+      newbib.name = "bibitem"
+      newbib["hidden"] = "true"
+      newbib&.at("./*[local-name() = 'ext']")&.remove
+      newbib["id"] = bib["id"]
+      newbib
+    end
+
+    def get_bibitem_docid(bib, identifier)
+      # IDs for repo references are untyped by default
+      docid = bib.at(ns("./docidentifier[not(@type)]")) ||
+        bib.at(ns("./docidentifier"))
+      docid &&= docid_prefix(docid)
+      if @files.get(docid) then docid
+      else
+        fail_update_bibitem(docid, identifier)
+        nil
+      end
+    end
+
+    def hide_refs(docxml)
+      docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or " \
+                      "@hidden = 'false'])]")).each do |f|
+        f["hidden"] = "true"
+      end
+    end
+
+    def strip_eref(eref)
+      eref.xpath(ns("./locality | ./localityStack")).each(&:remove)
+      eref.replace(eref.children)
+    end
+
+    def docid_to_citeas(bib)
+      docid = bib.at(ns("./docidentifier[@primary = 'true']")) ||
+        bib.at(ns("./docidentifier")) or return
+      docid_prefix(docid)
+    end
+
+    def collect_erefs(docxml)
+      docxml.xpath(ns("//eref"))
+        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
+        m[:citeas][i["citeas"]] = true
+        m[:bibitemid][i["bibitemid"]] = true
+      end
+    end
+
     private
 
+    def docid_prefix(docid)
+      type = docid["type"]
+      type == "metanorma-collection" and type = nil
+      @c.decode(@isodoc
+          .docid_prefix(type, docid.children.to_xml)).gsub(/\s/, " ")
+    end
+
     def create_non_existing_directory(output_directory)
       !File.exist?(output_directory) and
         FileUtils.mkdir_p(output_directory)
@@ -81,25 +135,28 @@ module Metanorma
       isodoc
     end
 
+    # create the @meta class of isodoc, for populating Liquid,
+    # with "navigation" set to the index bar.
+    # extracted from the manifest
     def isodoc_populate
-      # create the @meta class of isodoc, for populating Liquid,
-      # with "navigation" set to the index bar.
-      # extracted from the manifest
-      @isodoc.meta.set(:navigation, indexfile(@xml.at(ns("//manifest"))))
-      @isodoc.meta.set(:docrefs, liquid_docrefs)
-      @isodoc.meta.set(:"prefatory-content",
-                       isodoc_builder(@isodoc,
-                                      @xml.at(ns("//prefatory-content"))))
-      @isodoc.meta.set(:"final-content",
-                       isodoc_builder(isodoc,
-                                      @xml.at(ns("//final-content"))))
       @isodoc.info(@xml, nil)
+      m = @xml.at(ns("//manifest"))
+      { navigation: indexfile(m), nav_object: index_object(m),
+        docrefs: liquid_docrefs,
+        "prefatory-content": isodoc_builder(@xml.at(ns("//prefatory-content"))),
+        "final-content": isodoc_builder(@xml.at(ns("//final-content"))),
+        doctitle: m.at(ns("../bibdata/title"))&.text,
+        docnumber: m.at(ns("../bibdata/docidentifier"))&.text }.each do |k, v|
+        v and @isodoc.meta.set(
+          k, v
+        )
+      end
     end
 
-    def isodoc_builder(isodoc, node)
+    def isodoc_builder(node)
       Nokogiri::HTML::Builder.new(encoding: "UTF-8") do |b|
         b.div do |div|
-          node&.children&.each { |n| isodoc.parse(n, div) }
+          node&.children&.each { |n| @isodoc.parse(n, div) }
         end
       end.doc.root.to_html
     end

data/lib/metanorma/collection_renderer.rb

@@ -11,7 +11,7 @@ module Metanorma
   class CollectionRenderer
     FORMATS = %i[html xml doc pdf].freeze
 
-    attr_accessor :isodoc
+    attr_accessor :isodoc, :nested
     attr_reader :xml, :compile, :compile_options, :documents
 
     # This is only going to render the HTML collection
@@ -49,6 +49,9 @@ module Metanorma
       @compile = Compile.new
       @c = HTMLEntities.new
       @files_to_delete = []
+      @nested = options[:nested] # if false, this is the root instance of Renderer
+      # if true, then this is not the last time Renderer will be run
+      # (e.g. this is sectionsplit)
 
       # list of files in the collection
       @files = Metanorma::FileLookup.new(folder, self)
@@ -132,9 +135,7 @@ module Metanorma
     # @param elm [Nokogiri::XML::Element]
     # @return [String]
     def indexfile_title(elm)
-      lvl = elm.at(ns("./level"))&.text&.capitalize
-      lbl = elm.at(ns("./title"))&.text
-      "#{lvl}#{lvl && lbl ? ': ' : ''}#{lbl}"
+      elm.at(ns("./title"))&.text
     end
 
     # uses the identifier to label documents; other attributes (title) can be
@@ -172,10 +173,6 @@ module Metanorma
     end
 
     # single level navigation list, with hierarchical nesting
-    # if multiple lists are needed as separate HTML fragments, multiple
-    # instances of this function will be needed,
-    # and associated to different variables in the call to @isodoc.metadata_init
-    # (including possibly an array of HTML fragments)
     #
     # @param elm [Nokogiri::XML::Element]
     # @return [String] XML
@@ -191,6 +188,20 @@ module Metanorma
       end.doc.root.to_html
     end
 
+    # object to construct navigation out of in Liquid
+    def index_object(elm)
+      c = elm.xpath(ns("./manifest")).each_with_object([]) do |d, b|
+        b << index_object(d)
+      end
+      c.empty? and c = nil
+      r = Nokogiri::HTML::Builder.new do |b|
+        indexfile_docref(elm, b)
+      end
+      r &&= r.doc.root&.to_html&.gsub("\n", " ")
+      { title: indexfile_title(elm),
+        docrefs: r, children: c }.compact
+    end
+
     def liquid_docrefs
       @xml.xpath(ns("//docref[@index = 'true']")).each_with_object([]) do |d, m|
         ident = d.at(ns("./identifier")).children.to_xml

data/lib/metanorma/files_lookup.rb

@@ -26,6 +26,7 @@ module Metanorma
     end
 
     def read_files
+      @disambig = Util::DisambigFiles.new
       @xml.xpath(ns("//docref")).each { |d| read_file(d) }
     end
 
@@ -38,17 +39,16 @@ module Metanorma
       @files[i] = entry
     end
 
-    def bibdata_process(entry, identifier)
+    def bibdata_process(entry, ident)
       if entry[:attachment]
-        entry[:bibdata] = Metanorma::Document
-          .attachment_bibitem(identifier).root
+        entry[:bibdata] = Metanorma::Document.attachment_bibitem(ident).root
       else
         file, _filename = targetfile(entry, read: true)
         xml = Nokogiri::XML(file, &:huge)
-        add_document_suffix(identifier, xml)
-        entry[:anchors] = read_anchors(xml)
-        entry[:ids] = read_ids(xml)
-        entry[:bibdata] = xml.at(ns("//bibdata"))
+        add_document_suffix(ident, xml)
+        entry.merge!(anchors: read_anchors(xml), ids: read_ids(xml),
+                     bibdata: xml.at(ns("//bibdata")),
+                     document_suffix: xml.root["document_suffix"])
       end
     end
 
@@ -65,30 +65,33 @@ module Metanorma
     # identifier is the id with only spaces, no nbsp
     def file_entry(ref, identifier)
       out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
+      out1 = @disambig.source2dest_filename(out)
       ret = if ref["fileref"]
               { type: "fileref", ref: @documents[identifier].file,
-                rel_path: ref["fileref"], out_path: out }
-            else { type: "id", ref: ref["id"] } end
-      %w(attachment sectionsplit index presentation-xml
-         bare-after-first).each do |s|
-        ret[s.gsub("-", "").to_sym] = ref[s] if ref[s]
-      end
+                rel_path: ref["fileref"],
+                out_path: out1 } # @disambig.source2dest_filename(out) }
+            else { type: "id", ref: ref["id"] }
+            end
+      file_entry_copy(ref, ret)
       ret.compact
     end
 
-    def add_suffix_to_attributes(doc, suffix, tag_name, attribute_name)
-      doc.xpath(ns("//#{tag_name}[@#{attribute_name}]")).each do |elem|
-        elem.attributes[attribute_name].value =
-          "#{elem.attributes[attribute_name].value}_#{suffix}"
+    def file_entry_copy(ref, ret)
+      %w(attachment sectionsplit index presentation-xml
+         bare-after-first).each do |s|
+        ret[s.gsub("-", "").to_sym] = ref[s] if ref[s]
       end
     end
 
     def add_document_suffix(identifier, doc)
       document_suffix = Metanorma::Utils::to_ncname(identifier)
       Metanorma::Utils::anchor_attributes.each do |(tag_name, attribute_name)|
-        add_suffix_to_attributes(doc, document_suffix, tag_name, attribute_name)
+        Util::add_suffix_to_attributes(doc, document_suffix, tag_name,
+                                       attribute_name, @isodoc)
       end
       url_in_css_styles(doc, document_suffix)
+      doc.root["document_suffix"] ||= ""
+      doc.root["document_suffix"] += document_suffix
     end
 
     # update relative URLs, url(#...), in CSS in @style attrs (including SVG)

data/lib/metanorma/files_lookup_sectionsplit.rb

@@ -4,16 +4,30 @@ module Metanorma
     def add_section_split
       ret = @files.keys.each_with_object({}) do |k, m|
         if @files[k][:sectionsplit] == "true" && !@files[k]["attachment"]
-          s, manifest = sectionsplit(@files[k][:ref], k)
-          s.each_with_index { |f1, i| add_section_split_instance(f1, m, k, i) }
-          m["#{k}:index.html"] = add_section_split_cover(manifest, k)
-          @files_to_delete << m["#{k}:index.html"][:ref]
+          process_section_split_instance(k, m)
+          cleanup_section_split_instance(k, m)
         end
         m[k] = @files[k]
       end
       @files = ret
     end
 
+    def process_section_split_instance(key, manifest)
+      s, sectionsplit_manifest = sectionsplit(@files[key][:ref],
+                                              @files[key][:out_path], key)
+      s.each_with_index do |f1, i|
+        add_section_split_instance(f1, manifest, key, i)
+      end
+      manifest["#{key}:index.html"] =
+        add_section_split_cover(sectionsplit_manifest, key)
+    end
+
+    def cleanup_section_split_instance(key, manifest)
+      @files_to_delete << manifest["#{key}:index.html"][:ref]
+      #@files[key].delete(:ids).delete(:anchors)
+      @files[key][:indirect_key] = @sectionsplit.key
+    end
+
     def add_section_split_cover(manifest, ident)
       cover = @sectionsplit
         .section_split_cover(manifest, @parent.dir_name_cleanse(ident),
@@ -54,10 +68,10 @@ module Metanorma
       [presfile, newkey, xml]
     end
 
-    def sectionsplit(file, ident)
+    def sectionsplit(file, outfile, ident)
       @sectionsplit = Sectionsplit
-        .new(input: file, base: File.basename(file), dir: File.dirname(file),
-             output: file, compile_options: @parent.compile_options,
+        .new(input: file, base: outfile, dir: File.dirname(file),
+             output: outfile, compile_opts: @parent.compile_options,
              fileslookup: self, ident: ident, isodoc: @isodoc)
       coll = @sectionsplit.sectionsplit.sort_by { |f| f[:order] }
       xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"), &:huge)

data/lib/metanorma/processor.rb

@@ -18,7 +18,8 @@ module Metanorma
     end
 
     def input_to_isodoc(file, filename, options = {})
-      Metanorma::Input::Asciidoc.new.process(file, filename, @asciidoctor_backend, options)
+      Metanorma::Input::Asciidoc.new.process(file, filename,
+                                             @asciidoctor_backend, options)
     end
 
     # def input_to_isodoc(file, filename)
@@ -33,12 +34,17 @@ module Metanorma
       end
     end
 
+    def options_preprocess(options)
+      options[:output_formats] = output_formats
+    end
+
     def output(isodoc_node, _inname, outname, _format, _options = {})
       File.open(outname, "w:UTF-8") { |f| f.write(isodoc_node) }
     end
 
     def extract_options(file)
       Metanorma::Input::Asciidoc.new.extract_options(file)
+        .merge(output_formats: output_formats)
     end
 
     def extract_metanorma_options(file)

data/lib/metanorma/sectionsplit.rb

@@ -4,7 +4,7 @@ require_relative "sectionsplit_links"
 
 module Metanorma
   class Sectionsplit
-    attr_accessor :filecache
+    attr_accessor :filecache, :key
 
     def initialize(opts)
       @input_filename = opts[:input]
@@ -24,7 +24,7 @@ module Metanorma
 
     def build_collection
       collection_setup(@base, @dir)
-      files = sectionsplit #(@input_filename, @base, @dir, @compile_opts)
+      files = sectionsplit # (@input_filename, @base, @dir, @compile_opts)
       input_xml = Nokogiri::XML(File.read(@input_filename,
                                           encoding: "UTF-8"), &:huge)
       collection_manifest(@base, files, input_xml, @xml, @dir).render(
@@ -50,7 +50,7 @@ module Metanorma
 
     def coll_cover
       <<~COVER
-        <html><head/><body>
+        <html><head><meta charset="UTF-8"/></head><body>
               <h1>{{ doctitle }}</h1>
               <h2>{{ docnumber }}</h2>
               <nav>{{ navigation }}</nav>
@@ -71,7 +71,8 @@ module Metanorma
       @key = xref_preprocess(xml, @fileslookup, @ident)
       SPLITSECTIONS.each_with_object([]) do |n, ret|
         conflate_floatingtitles(xml.xpath(ns(n[0]))).each do |s|
-          ret << sectionfile(xml, emptydoc(xml), "#{@base}.#{ret.size}", s, n[1])
+          ret << sectionfile(xml, emptydoc(xml), "#{@base}.#{ret.size}", s,
+                             n[1])
         end
       end
     end
@@ -107,17 +108,30 @@ module Metanorma
     def sectionsplit_preprocess_semxml(file, filename)
       xml = Nokogiri::XML(file, &:huge)
       type = xml.root.name.sub("-standard", "").to_sym
-      @fileslookup&.parent&.update_xrefs(xml, @ident, {})
-      xml1 = Tempfile.open([filename, ".xml"], encoding: "utf-8") do |f|
-        #f.write(@isodoc.to_xml(svg_preprocess(xml)))
-        f.write(@isodoc.to_xml((xml)))
-        f
-      end
+      sectionsplit_update_xrefs(xml)
+      xml1 = sectionsplit_write_semxml(filename, xml)
       @filecache ||= []
       @filecache << xml1
       [xml1.path, type]
     end
 
+    def sectionsplit_update_xrefs(xml)
+      if c = @fileslookup&.parent
+        n = c.nested
+        c.nested = true # so unresolved erefs are not deleted
+        c.update_xrefs(xml, @ident, {})
+        c.nested = n
+      end
+    end
+
+    def sectionsplit_write_semxml(filename, xml)
+      Tempfile.open([filename, ".xml"], encoding: "utf-8") do |f|
+        # f.write(@isodoc.to_xml(svg_preprocess(xml)))
+        f.write(@isodoc.to_xml(xml))
+        f
+      end
+    end
+
     def emptydoc(xml)
       out = xml.dup
       out.xpath(

data/lib/metanorma/sectionsplit_links.rb

@@ -3,27 +3,29 @@ module Metanorma
     def xref_preprocess(xml, _fileslookup, _identifier)
       key = (0...8).map { rand(65..90).chr }.join # random string
       xml.root["type"] = key # to force recognition of internal refs
+      # bookmarks etc as new id elements introduced in Presentation XML:
+      # add doc suffix
+      Metanorma::Utils::anchor_attributes.each do |(tag_name, attribute_name)|
+        Util::add_suffix_to_attributes(xml, xml.root["document_suffix"],
+                                       tag_name, attribute_name, @isodoc)
+      end
       key
     end
 
     def xref_process(section, xml, key)
       svg_preprocess(section, Metanorma::Utils::to_ncname(@ident))
       refs = eref_to_internal_eref(section, xml, key)
-      refs += xref_to_internal_eref(section, key)
+      refs += xref_to_internal_eref(section, xml, key)
       ins = new_hidden_ref(section)
       copied_refs = copy_repo_items_biblio(ins, section, xml)
-      insert_indirect_biblio(ins, refs - copied_refs, key)
+      insert_indirect_biblio(ins, refs - copied_refs, key, xml)
     end
 
-    def svg_preprocess(xml, document_suffix)
+    def svg_preprocess(xml, doc_suffix)
+      suffix = doc_suffix.nil? || doc_suffix.blank? ? "" : "_#{doc_suffix}"
       xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
         m = svgmap_wrap(s)
-        s.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
-          /^#/.match? a["href"] or next
-          a["href"] = a["href"].sub(/^#/, "")
-          m << "<target href='#{a['href']}'>" \
-               "<xref target='#{a['href']}_#{document_suffix}'/></target>"
-        end
+        svg_xrefs(s, m, suffix)
       end
       xml
     end
@@ -35,33 +37,75 @@ module Metanorma
       svg.at("./ancestor::xmlns:svgmap")
     end
 
+    def svg_xrefs(svg, svgmap, suffix)
+      svg.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
+        /^#/.match? a["href"] or next
+        a["href"] = a["href"].sub(/^#/, "")
+        svgmap << "<target href='#{a['href']}'>" \
+                  "<xref target='#{a['href']}#{suffix}'/></target>"
+      end
+    end
+
     def make_anchor(anchor)
       "<localityStack><locality type='anchor'><referenceFrom>" \
         "#{anchor}</referenceFrom></locality></localityStack>"
     end
 
-    def xref_to_internal_eref(xml, key)
-      xml.xpath(ns("//xref")).each_with_object({}) do |x, m|
-        x["bibitemid"] = "#{key}_#{x['target']}"
-        x << make_anchor(x["target"])
+    def xref_to_internal_eref(section, xml, key)
+      bibitems, indirect = xref_to_internal_eref_prep(section, xml)
+      section.xpath(ns("//xref")).each_with_object({}) do |x, m|
+        xref_prefix_key(x, key, indirect)
+        x["bibitemid"] = x["target"]
         m[x["bibitemid"]] = true
-        x.delete("target")
-        x["type"] = key
-        x.name = "eref"
+        xref_to_internal_eref_anchor(x, key, bibitems, xml.root["document_suffix"])
       end.keys
     end
 
+    def xref_to_internal_eref_prep(section, xml)
+      bibitems = Util::gather_bibitems(section)
+      indirect_bibitems = Util::gather_bibitems(xml)
+        .select { |_, v| indirect_bib?(v) }
+      [bibitems, indirect_bibitems]
+    end
+
+    def xref_to_internal_eref_anchor(xref, key, bibitems, document_suffix)
+      t = xref["target"]
+      if d = bibitems[t]&.at(ns("./docidentifier[@type = 'repository']"))
+        m = %r{^([^/]+)}.match(d.text) and
+          t.sub!(%r(#{m[0]}_), "")
+      end
+      t.sub!(%r{^#{key}_}, "")
+      xref << make_anchor(t.sub(%r(_#{document_suffix}$), ""))
+      xref.delete("target")
+      xref.name = "eref"
+    end
+
+    def xref_prefix_key(xref, key, indirect)
+      if b = indirect[xref["target"]]
+        t = b.at(ns("./docidentifier[@type = 'repository']"))
+        xref["type"] = t.text.sub(%r{/.*$}, "")
+      else
+        xref["target"] = "#{key}_#{xref['target']}"
+        xref["type"] = key
+      end
+    end
+
     def eref_to_internal_eref(section, xml, key)
-      bibitems = Util::gather_bibitems(xml)
-      bibitemids = Util::gather_bibitemids(section)
+      bibitems, bibitemids = eref_to_internal_eref_prep(section, xml)
       eref_to_internal_eref_select(section, xml, bibitems)
         .each_with_object([]) do |x, m|
-        url = bibitems[x]&.at(ns("./uri[@type = 'citation']"))
-        bibitemids[x]&.each do |e|
-          id = eref_to_internal_eref1(e, key, url)
-          id and m << id
+          url = bibitems[x]&.at(ns("./uri[@type = 'citation']"))
+          bibitemids[x]&.each do |e|
+            id = eref_to_internal_eref1(e, key, url) and m << id
+          end
         end
-      end
+    end
+
+    def eref_to_internal_eref_prep(section, xml)
+      bibitems = Util::gather_bibitems(xml)
+        .delete_if { |_, v| internal_bib?(v) }
+      bibitemids = Util::gather_bibitemids(section)
+      [bibitems, bibitemids]
     end
 
     def eref_to_internal_eref1(elem, key, url)
@@ -80,11 +124,22 @@ module Metanorma
     def eref_to_internal_eref_select(section, _xml, bibitems)
       refs = Util::gather_bibitemids(section).keys
       refs.uniq.reject do |x|
-        b = bibitems[x] and (b["type"] == "internal" ||
-                             b.at(ns("./docidentifier/@type = 'repository']")))
+        b = bibitems[x] and ( indirect_bib?(b) || internal_bib?(b) )
       end
     end
 
+    def internal_bib?(bibitem)
+      bibitem["type"] == "internal" ||
+        bibitem.at(ns("./docidentifier[@type = 'repository']"))
+    end
+
+    def indirect_bib?(bibitem)
+      a = bibitem.at(ns("./docidentifier[@type = 'repository']")) or
+        return false
+      %r{^current-metanorma-collection/}.match?(a.text) and return false
+      a.text.include?("/")
+    end
+
     # from standoc
     def new_hidden_ref(xmldoc)
       ins = xmldoc.at("bibliography") or
@@ -96,21 +151,39 @@ module Metanorma
       bibitems = Util::gather_bibitems(section)
       xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
         .each_with_object([]) do |b, m|
-        bibitems[b["id"]] or next
-        # section.at("//*[@bibitemid = '#{b['id']}']") or next
-        ins << b.dup
-        m << b["id"]
-      end
+          bibitems[b["id"]] or next
+          # section.at("//*[@bibitemid = '#{b['id']}']") or next
+          ins << b.dup
+          m << b["id"]
+        end
     end
 
-    def insert_indirect_biblio(ins, refs, prefix)
-      refs.each do |x|
-        ins << <<~BIBENTRY
-          <bibitem id="#{x}" type="internal">
-          <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
-          </bibitem>
-        BIBENTRY
+    def insert_indirect_biblio(ins, refs, key, xml)
+      refs.empty? and return
+      internal_bibitems, external_bibitems = insert_indirect_biblio_prep(xml)
+      refs.compact.reject do |x|
+        #external_bibitems[x.sub(/^#{key}_/, "")]
+      end.each do |x|
+        ins << if b = internal_bibitems[x.sub(/^#{key}_/, "")]
+                 b.dup.tap { |m| m["id"] = x }
+               else new_indirect_bibitem(x, key)
+               end
       end
     end
+
+    def insert_indirect_biblio_prep(xml)
+      bibitems = Util::gather_bibitems(xml)
+      internal_bibitems = bibitems.select { |_, v| internal_bib?(v) }
+      external_bibitems = bibitems.reject { |_, v| internal_bib?(v) }
+      [internal_bibitems, external_bibitems]
+    end
+
+    def new_indirect_bibitem(ident, prefix)
+      <<~BIBENTRY
+        <bibitem id="#{ident}" type="internal">
+        <docidentifier type="repository">#{ident.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+        </bibitem>
+      BIBENTRY
+    end
   end
 end

data/lib/metanorma/util.rb

@@ -60,13 +60,26 @@ module Metanorma
       end
     end
 
+    def self.add_suffix_to_attributes(doc, suffix, tag_name, attr_name, isodoc)
+      (suffix.nil? || suffix.empty?) and return
+      doc.xpath(isodoc.ns("//#{tag_name}[@#{attr_name}]")).each do |elem|
+        a = elem.attributes[attr_name].value
+        /_#{suffix}$/.match?(a) or
+          elem.attributes[attr_name].value = "#{a}_#{suffix}"
+      end
+    end
+
     class DisambigFiles
       def initialize
         @seen_filenames = []
       end
 
+      def strip_root(name)
+        name.sub(%r{^(\./)?(\.\./)+}, "")
+      end
+
       def source2dest_filename(name, disambig = true)
-        n = name.sub(%r{^(\./)?(\.\./)+}, "")
+        n = strip_root(name)
         dir = File.dirname(n)
         base = File.basename(n)
         if disambig && @seen_filenames.include?(base)

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.6.4".freeze
+  VERSION = "1.6.6".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.6.4
+  version: 1.6.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-09 00:00:00.000000000 Z
+date: 2023-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor