metanorma 1.6.2 → 1.6.4

data/lib/metanorma/files_lookup.rb

@@ -1,10 +1,13 @@
 require "isodoc"
 require "htmlentities"
 require "metanorma-utils"
+require_relative "files_lookup_sectionsplit"
 
 module Metanorma
   # XML collection renderer
   class FileLookup
+    attr_accessor :files_to_delete, :parent
+
     # hash for each document in collection of document identifier to:
     # document reference (fileref or id), type of document reference,
     # and bibdata entry for that file
@@ -18,12 +21,12 @@ module Metanorma
       @path = path
       @compile = parent.compile
       @documents = parent.documents
+      @files_to_delete = []
       read_files
     end
 
     def read_files
       @xml.xpath(ns("//docref")).each { |d| read_file(d) }
-      add_section_split
     end
 
     def read_file(docref)
@@ -41,9 +44,10 @@ module Metanorma
           .attachment_bibitem(identifier).root
       else
         file, _filename = targetfile(entry, read: true)
-        xml = Nokogiri::XML(file)
+        xml = Nokogiri::XML(file, &:huge)
         add_document_suffix(identifier, xml)
         entry[:anchors] = read_anchors(xml)
+        entry[:ids] = read_ids(xml)
         entry[:bibdata] = xml.at(ns("//bibdata"))
       end
     end
@@ -55,69 +59,6 @@ module Metanorma
       entry[:bibitem].at("./*[local-name() = 'ext']")&.remove
     end
 
-    def add_section_split
-        #require "debug"; binding.b
-      ret = @files.keys.each_with_object({}) do |k, m|
-        if @files[k][:sectionsplit] == "true" && !@files[k]["attachment"]
-          s, manifest = sectionsplit(@files[k][:ref])
-          s.each_with_index do |f1, i|
-            add_section_split_instance(f1, m, k, i)
-          end
-          m["#{k}:index.html"] = add_section_split_cover(manifest, k)
-        end
-        m[k] = @files[k]
-        #require "debug"; binding.b
-      end
-      @files = ret
-    end
-
-    def add_section_split_cover(manifest, ident)
-      cover = section_split_cover(manifest, @parent.dir_name_cleanse(ident))
-      @files[ident][:out_path] = cover
-      { attachment: true, index: false, out_path: cover,
-        ref: File.join(File.dirname(manifest.file), cover) }
-    end
-
-    def section_split_cover(col, ident)
-      dir = File.dirname(col.file)
-      @compile.collection_setup(nil, dir)
-      #require "debug";binding.b
-      CollectionRenderer.new(col, dir,
-                             output_folder: "#{ident}_collection",
-                             format: %i(html),
-                             coverpage: File.join(dir, "cover.html")).coverpage
-      FileUtils.mv "#{ident}_collection/index.html",
-                   File.join(dir, "#{ident}_index.html")
-      FileUtils.rm_rf "#{ident}_collection"
-      "#{ident}_index.html"
-    end
-
-    def add_section_split_instance(file, manifest, key, idx)
-      presfile = File.join(File.dirname(@files[key][:ref]),
-                           File.basename(file[:url]))
-      newkey = key("#{key.strip} #{file[:title]}")
-      manifest[newkey] =
-        { parentid: key, presentationxml: true, type: "fileref",
-          rel_path: file[:url], out_path: File.basename(file[:url]),
-          anchors: read_anchors(Nokogiri::XML(File.read(presfile))),
-          bibdata: @files[key][:bibdata], ref: presfile }
-      manifest[newkey][:bare] = true unless idx.zero?
-    end
-
-    def sectionsplit(file)
-      #require "debug"; binding.b
-      @compile.compile(
-        file, { format: :asciidoc, extension_keys: [:presentation] }
-        .merge(@parent.compile_options)
-      )
-      r = file.sub(/\.xml$/, ".presentation.xml")
-      xml = Nokogiri::XML(File.read(r))
-      s = @compile.sectionsplit(xml, File.basename(r), File.dirname(r))
-        .sort_by { |f| f[:order] }
-      [s, @compile.collection_manifest(File.basename(r), s, xml, nil,
-                                       File.dirname(r))]
-    end
-
     # rel_path is the source file address, determined relative to the YAML.
     # out_path is the destination file address, with any references outside
     # the working directory (../../...) truncated
@@ -220,8 +161,19 @@ module Metanorma
       ret[val[:type]][val[:value]] = key if val[:value]
     end
 
+    # Also parse all ids in doc (including ones which won't be xref targets)
+    def read_ids(xml)
+      ret = {}
+      xml.traverse do |x|
+        x.text? and next
+        /^semantic__/.match?(x.name) and next
+        x["id"] and ret[x["id"]] = true
+      end
+      ret
+    end
+
     def key(ident)
-      @c.decode(ident).gsub(/(\s|[ ])+/, " ")
+      @c.decode(ident).gsub(/(\p{Zs})+/, " ").sub(/^metanorma-collection /, "")
     end
 
     def keys
@@ -229,10 +181,8 @@ module Metanorma
     end
 
     def get(ident, attr = nil)
-      if attr
-        @files[key(ident)][attr]
-      else
-        @files[key(ident)]
+      if attr then @files[key(ident)][attr]
+      else @files[key(ident)]
       end
     end
 

data/lib/metanorma/files_lookup_sectionsplit.rb

@@ -0,0 +1,69 @@
+module Metanorma
+  # XML collection renderer
+  class FileLookup
+    def add_section_split
+      ret = @files.keys.each_with_object({}) do |k, m|
+        if @files[k][:sectionsplit] == "true" && !@files[k]["attachment"]
+          s, manifest = sectionsplit(@files[k][:ref], k)
+          s.each_with_index { |f1, i| add_section_split_instance(f1, m, k, i) }
+          m["#{k}:index.html"] = add_section_split_cover(manifest, k)
+          @files_to_delete << m["#{k}:index.html"][:ref]
+        end
+        m[k] = @files[k]
+      end
+      @files = ret
+    end
+
+    def add_section_split_cover(manifest, ident)
+      cover = @sectionsplit
+        .section_split_cover(manifest, @parent.dir_name_cleanse(ident),
+                             one_doc_collection?)
+      @files[ident][:out_path] = cover
+      { attachment: true, index: false, out_path: cover,
+        ref: File.join(File.dirname(manifest.file), cover) }
+    end
+
+    def one_doc_collection?
+      docs = 0
+      @files.each_value do |v|
+        v[:attachment] and next
+        v[:presentationxml] and next
+        docs += 1
+      end
+      docs > 1
+    end
+
+    def add_section_split_instance(file, manifest, key, idx)
+      presfile, newkey, xml =
+        add_section_split_instance_prep(file, key)
+      manifest[newkey] =
+        { parentid: key, presentationxml: true, type: "fileref",
+          rel_path: file[:url], out_path: File.basename(file[:url]),
+          anchors: read_anchors(xml), ids: read_ids(xml),
+          sectionsplit_output: true,
+          bibdata: @files[key][:bibdata], ref: presfile }
+      @files_to_delete << file[:url]
+      manifest[newkey][:bare] = true unless idx.zero?
+    end
+
+    def add_section_split_instance_prep(file, key)
+      presfile = File.join(File.dirname(@files[key][:ref]),
+                           File.basename(file[:url]))
+      newkey = key("#{key.strip} #{file[:title]}")
+      xml = Nokogiri::XML(File.read(presfile), &:huge)
+      [presfile, newkey, xml]
+    end
+
+    def sectionsplit(file, ident)
+      @sectionsplit = Sectionsplit
+        .new(input: file, base: File.basename(file), dir: File.dirname(file),
+             output: file, compile_options: @parent.compile_options,
+             fileslookup: self, ident: ident, isodoc: @isodoc)
+      coll = @sectionsplit.sectionsplit.sort_by { |f| f[:order] }
+      xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"), &:huge)
+      [coll, @sectionsplit
+        .collection_manifest(File.basename(file), coll, xml, nil,
+                             File.dirname(file))]
+    end
+  end
+end

data/lib/metanorma/input/asciidoc.rb

@@ -69,7 +69,8 @@ module Metanorma
            pdf-allow-print pdf-allow-print-hq pdf-allow-fill-in-forms
            fonts font-license-agreement pdf-allow-access-content
            pdf-encrypt-metadata iso-word-template document-scheme
-           localize-number iso-word-bg-strip-color modspec-identifier-base).freeze
+           localize-number iso-word-bg-strip-color modspec-identifier-base)
+          .freeze
 
       EMPTY_ADOC_OPTIONS_DEFAULT_TRUE =
         %w(data-uri-image suppress-asciimath-dup use-xinclude
@@ -80,7 +81,7 @@ module Metanorma
            toc-tables toc-recommendations).freeze
 
       def attr_name_normalise(name)
-        name.gsub(/-/, "").sub(/override$/, "_override").sub(/pdf$/, "_pdf")
+        name.gsub("-", "").sub(/override$/, "_override").sub(/pdf$/, "_pdf")
           .to_sym
       end
 

data/lib/metanorma/sectionsplit.rb

@@ -1,31 +1,35 @@
 require "yaml"
+require_relative "util"
+require_relative "sectionsplit_links"
 
 module Metanorma
-  class Compile
-    # assume we pass in Presentation XML, but we want to recover Semantic XML
-    def sectionsplit_convert(input_filename, file, output_filename = nil,
-                             opts = {})
-      @isodoc = IsoDoc::Convert.new({})
-      input_filename += ".xml" unless input_filename.match?(/\.xml$/)
-      File.exist?(input_filename) or
-        File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
-      presxml = File.read(input_filename, encoding: "utf-8")
-      @openmathdelim, @closemathdelim = @isodoc.extract_delims(presxml)
-      xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
-      build_collection(xml, presxml, output_filename || filename, dir, opts)
+  class Sectionsplit
+    attr_accessor :filecache
+
+    def initialize(opts)
+      @input_filename = opts[:input]
+      @base = opts[:base]
+      @output_filename = opts[:output]
+      @xml = opts[:xml]
+      @dir = opts[:dir]
+      @compile_opts = opts[:compile_opts] || {}
+      @fileslookup = opts[:fileslookup]
+      @ident = opts[:ident]
+      @isodoc = opts[:isodoc]
     end
 
     def ns(xpath)
       @isodoc.ns(xpath)
     end
 
-    def build_collection(xml, presxml, filename, dir, opts = {})
-      base = File.basename(filename)
-      collection_setup(base, dir)
-      files = sectionsplit(xml, base, dir)
-      collection_manifest(base, files, xml, presxml, dir).render(
-        { format: %i(html), output_folder: "#{filename}_collection",
-          coverpage: File.join(dir, "cover.html") }.merge(opts),
+    def build_collection
+      collection_setup(@base, @dir)
+      files = sectionsplit #(@input_filename, @base, @dir, @compile_opts)
+      input_xml = Nokogiri::XML(File.read(@input_filename,
+                                          encoding: "UTF-8"), &:huge)
+      collection_manifest(@base, files, input_xml, @xml, @dir).render(
+        { format: %i(html), output_folder: "#{@output_filename}_collection",
+          coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
       )
     end
 
@@ -46,13 +50,11 @@ module Metanorma
 
     def coll_cover
       <<~COVER
-        <html><head/>
-            <body>
+        <html><head/><body>
               <h1>{{ doctitle }}</h1>
               <h2>{{ docnumber }}</h2>
               <nav>{{ navigation }}</nav>
-            </body>
-        </html>
+            </body></html>
       COVER
     end
 
@@ -62,161 +64,92 @@ module Metanorma
        ["//bibliography/*[not(@hidden = 'true')]", "bibliography"],
        ["//indexsect", nil], ["//colophon", nil]].freeze
 
-    def sectionsplit(xml, filename, dir)
-      @key = xref_preprocess(xml)
-      @splitdir = dir
-      out = emptydoc(xml)
+    # Input XML is Semantic
+    # def sectionsplit(filename, basename, dir, compile_options, fileslookup = nil, ident = nil)
+    def sectionsplit
+      xml = sectionsplit_prep(File.read(@input_filename), @base, @dir)
+      @key = xref_preprocess(xml, @fileslookup, @ident)
       SPLITSECTIONS.each_with_object([]) do |n, ret|
-        xml.xpath(ns(n[0])).each do |s|
-          ret << sectionfile(xml, out, "#{filename}.#{ret.size}", s, n[1])
-        end
-      end
-    end
-
-    def emptydoc(xml)
-      out = xml.dup
-      out.xpath(
-        ns("//preface | //sections | //annex | //bibliography/clause | " \
-           "//bibliography/references[not(@hidden = 'true')] | //indexsect" \
-           "//colophon"),
-      ).each(&:remove)
-      out
-    end
-
-    def sectionfile(fulldoc, xml, file, chunk, parentnode)
-      fname = create_sectionfile(fulldoc, xml.dup, file, chunk, parentnode)
-      { order: chunk["displayorder"].to_i, url: fname,
-        title: titlerender(chunk) }
-    end
-
-    def create_sectionfile(xml, out, file, chunk, parentnode)
-      ins = out.at(ns("//misccontainer")) || out.at(ns("//bibdata"))
-      if parentnode
-        ins.next = "<#{parentnode}/>"
-        ins.next.add_child(chunk.dup)
-      else ins.next = chunk.dup
-      end
-      xref_process(out, xml, @key)
-      outname = "#{file}.xml"
-      File.open(File.join(@splitdir, outname), "w:UTF-8") { |f| f.write(out) }
-      outname
-    end
-
-    def xref_preprocess(xml)
-      svg_preprocess(xml)
-      key = (0...8).map { rand(65..90).chr }.join # random string
-      xml.root["type"] = key # to force recognition of internal refs
-      key
-    end
-
-    def xref_process(section, xml, key)
-      refs = eref_to_internal_eref(section, xml, key)
-      refs += xref_to_internal_eref(section, key)
-      ins = new_hidden_ref(section)
-      copied_refs = copy_repo_items_biblio(ins, section, xml)
-      insert_indirect_biblio(ins, refs - copied_refs, key)
-    end
-
-    def svg_preprocess(xml)
-      xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
-        m = svgmap_wrap(s)
-        s.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
-          next unless /^#/.match? a["href"]
-
-          a["href"] = a["href"].sub(/^#/, "")
-          m << "<target href='#{a['href']}'>" \
-               "<xref target='#{a['href']}'/></target>"
+        conflate_floatingtitles(xml.xpath(ns(n[0]))).each do |s|
+          ret << sectionfile(xml, emptydoc(xml), "#{@base}.#{ret.size}", s, n[1])
         end
       end
     end
 
-    def svgmap_wrap(svg)
-      ret = svg.at("./ancestor::xmlns:svgmap") and return ret
-      ret = svg.at("./ancestor::xmlns:figure")
-      ret.wrap("<svgmap/>")
-      svg.at("./ancestor::xmlns:svgmap")
+    def block?(node)
+      %w(p table formula admonition ol ul dl figure quote sourcecode example
+         pre note pagebrreak hr bookmark requirement recommendation permission
+         svgmap inputform toc passthrough review imagemap).include?(node.name)
     end
 
-    def make_anchor(anchor)
-      "<localityStack><locality type='anchor'><referenceFrom>" \
-        "#{anchor}</referenceFrom></locality></localityStack>"
-    end
-
-    def xref_to_internal_eref(xml, key)
-      xml.xpath(ns("//xref")).each_with_object({}) do |x, m|
-        x["bibitemid"] = "#{key}_#{x['target']}"
-        x << make_anchor(x["target"])
-        m[x["bibitemid"]] = true
-        x.delete("target")
-        x["type"] = key
-        x.name = "eref"
-      end.keys
-    end
-
-    def eref_to_internal_eref(section, xml, key)
-      eref_to_internal_eref_select(section, xml).each_with_object([]) do |x, m|
-        url = xml.at(ns("//bibitem[@id = '#{x}']/uri[@type = 'citation']"))
-        section.xpath("//*[@bibitemid = '#{x}']").each do |e|
-          id = eref_to_internal_eref1(e, key, url)
-          id and m << id
+    def conflate_floatingtitles(nodes)
+      holdover = false
+      nodes.each_with_object([]) do |x, m|
+        if holdover then m.last << x
+        else m << [x]
         end
+        holdover = block?(x)
       end
     end
 
-    def eref_to_internal_eref1(elem, key, url)
-      if url
-        elem.name = "link"
-        elem["target"] = url
-        nil
-      else
-        elem["bibitemid"] = "#{key}_#{elem['bibitemid']}"
-        elem << make_anchor(elem["bibitemid"])
-        elem["type"] = key
-        elem["bibitemid"]
-      end
-    end
-
-    def eref_to_internal_eref_select(section, xml)
-      refs = section.xpath("//*/@bibitemid").map { |x| x.text } # rubocop:disable Style/SymbolProc
-      refs.uniq.reject do |x|
-        xml.at(ns("//bibitem[@id = '#{x}'][@type = 'internal']")) ||
-          xml.at(ns("//bibitem[@id = '#{x}']" \
-                    "[docidentifier/@type = 'repository']"))
+    def sectionsplit_prep(file, filename, dir)
+      @splitdir = dir
+      xml1filename, type = sectionsplit_preprocess_semxml(file, filename)
+      Compile.new.compile(
+        xml1filename,
+        { format: :asciidoc, extension_keys: [:presentation], type: type }
+       .merge(@compile_opts),
+      )
+      Nokogiri::XML(File.read(xml1filename.sub(/\.xml$/, ".presentation.xml"),
+                              encoding: "utf-8"), &:huge)
+    end
+
+    def sectionsplit_preprocess_semxml(file, filename)
+      xml = Nokogiri::XML(file, &:huge)
+      type = xml.root.name.sub("-standard", "").to_sym
+      @fileslookup&.parent&.update_xrefs(xml, @ident, {})
+      xml1 = Tempfile.open([filename, ".xml"], encoding: "utf-8") do |f|
+        #f.write(@isodoc.to_xml(svg_preprocess(xml)))
+        f.write(@isodoc.to_xml((xml)))
+        f
       end
+      @filecache ||= []
+      @filecache << xml1
+      [xml1.path, type]
     end
 
-    # from standoc
-    def new_hidden_ref(xmldoc)
-      ins = xmldoc.at("bibliography") or
-        xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
-      ins.add_child("<references hidden='true' normative='false'/>").first
+    def emptydoc(xml)
+      out = xml.dup
+      out.xpath(
+        ns("//preface | //sections | //annex | //bibliography/clause | " \
+           "//bibliography/references[not(@hidden = 'true')] | //indexsect | " \
+           "//colophon"),
+      ).each(&:remove)
+      out
     end
 
-    def copy_repo_items_biblio(ins, section, xml)
-      xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
-        .each_with_object([]) do |b, m|
-        section.at("//*[@bibitemid = '#{b['id']}']") or next
-        ins << b.dup
-        m << b["id"]
-      end
+    def sectionfile(fulldoc, xml, file, chunks, parentnode)
+      fname = create_sectionfile(fulldoc, xml.dup, file, chunks, parentnode)
+      { order: chunks.last["displayorder"].to_i, url: fname,
+        title: titlerender(chunks.last) }
     end
 
-    def insert_indirect_biblio(ins, refs, prefix)
-      refs.each do |x|
-        ins << <<~BIBENTRY
-          <bibitem id="#{x}" type="internal">
-          <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
-          </bibitem>
-        BIBENTRY
+    def create_sectionfile(xml, out, file, chunks, parentnode)
+      ins = out.at(ns("//metanorma-extension")) || out.at(ns("//bibdata"))
+      sectionfile_insert(ins, chunks, parentnode)
+      xref_process(out, xml, @key)
+      outname = "#{file}.xml"
+      File.open(File.join(@splitdir, outname), "w:UTF-8") do |f|
+        f.write(out)
       end
+      outname
     end
 
-    def recursive_string_keys(hash)
-      case hash
-      when Hash then hash.map { |k, v| [k.to_s, recursive_string_keys(v)] }.to_h
-      when Enumerable then hash.map { |v| recursive_string_keys(v) }
-      else
-        hash
+    def sectionfile_insert(ins, chunks, parentnode)
+      if parentnode
+        ins.next = "<#{parentnode}/>"
+        chunks.each { |c| ins.next.add_child(c.dup) }
+      else chunks.each { |c| ins.next = c.dup }
       end
     end
 
@@ -249,7 +182,20 @@ module Metanorma
           end
         },
       }
-      recursive_string_keys(ret).to_yaml
+      Util::recursive_string_keys(ret).to_yaml
+    end
+
+    def section_split_cover(col, ident, one_doc_coll)
+      dir = File.dirname(col.file)
+      collection_setup(nil, dir)
+      CollectionRenderer.new(col, dir,
+                             output_folder: "#{ident}_collection",
+                             format: %i(html),
+                             coverpage: File.join(dir, "cover.html")).coverpage
+      filename = one_doc_coll ? "#{ident}_index.html" : "index.html"
+      FileUtils.mv "#{ident}_collection/index.html", File.join(dir, filename)
+      FileUtils.rm_rf "#{ident}_collection"
+      filename
     end
   end
 end

data/lib/metanorma/sectionsplit_links.rb

@@ -0,0 +1,116 @@
+module Metanorma
+  class Sectionsplit
+    def xref_preprocess(xml, _fileslookup, _identifier)
+      key = (0...8).map { rand(65..90).chr }.join # random string
+      xml.root["type"] = key # to force recognition of internal refs
+      key
+    end
+
+    def xref_process(section, xml, key)
+      svg_preprocess(section, Metanorma::Utils::to_ncname(@ident))
+      refs = eref_to_internal_eref(section, xml, key)
+      refs += xref_to_internal_eref(section, key)
+      ins = new_hidden_ref(section)
+      copied_refs = copy_repo_items_biblio(ins, section, xml)
+      insert_indirect_biblio(ins, refs - copied_refs, key)
+    end
+
+    def svg_preprocess(xml, document_suffix)
+      xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
+        m = svgmap_wrap(s)
+        s.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
+          /^#/.match? a["href"] or next
+          a["href"] = a["href"].sub(/^#/, "")
+          m << "<target href='#{a['href']}'>" \
+               "<xref target='#{a['href']}_#{document_suffix}'/></target>"
+        end
+      end
+      xml
+    end
+
+    def svgmap_wrap(svg)
+      ret = svg.at("./ancestor::xmlns:svgmap") and return ret
+      ret = svg.at("./ancestor::xmlns:figure")
+      ret.wrap("<svgmap/>")
+      svg.at("./ancestor::xmlns:svgmap")
+    end
+
+    def make_anchor(anchor)
+      "<localityStack><locality type='anchor'><referenceFrom>" \
+        "#{anchor}</referenceFrom></locality></localityStack>"
+    end
+
+    def xref_to_internal_eref(xml, key)
+      xml.xpath(ns("//xref")).each_with_object({}) do |x, m|
+        x["bibitemid"] = "#{key}_#{x['target']}"
+        x << make_anchor(x["target"])
+        m[x["bibitemid"]] = true
+        x.delete("target")
+        x["type"] = key
+        x.name = "eref"
+      end.keys
+    end
+
+    def eref_to_internal_eref(section, xml, key)
+      bibitems = Util::gather_bibitems(xml)
+      bibitemids = Util::gather_bibitemids(section)
+      eref_to_internal_eref_select(section, xml, bibitems)
+        .each_with_object([]) do |x, m|
+        url = bibitems[x]&.at(ns("./uri[@type = 'citation']"))
+        bibitemids[x]&.each do |e|
+          id = eref_to_internal_eref1(e, key, url)
+          id and m << id
+        end
+      end
+    end
+
+    def eref_to_internal_eref1(elem, key, url)
+      if url
+        elem.name = "link"
+        elem["target"] = url
+        nil
+      else
+        elem["bibitemid"] = "#{key}_#{elem['bibitemid']}"
+        elem << make_anchor(elem["bibitemid"])
+        elem["type"] = key
+        elem["bibitemid"]
+      end
+    end
+
+    def eref_to_internal_eref_select(section, _xml, bibitems)
+      refs = Util::gather_bibitemids(section).keys
+      refs.uniq.reject do |x|
+        b = bibitems[x] and (b["type"] == "internal" ||
+                             b.at(ns("./docidentifier/@type = 'repository']")))
+      end
+    end
+
+    # from standoc
+    def new_hidden_ref(xmldoc)
+      ins = xmldoc.at("bibliography") or
+        xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+      ins.add_child("<references hidden='true' normative='false'/>").first
+    end
+
+    def copy_repo_items_biblio(ins, section, xml)
+      bibitems = Util::gather_bibitems(section)
+      xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
+        .each_with_object([]) do |b, m|
+        bibitems[b["id"]] or next
+        # section.at("//*[@bibitemid = '#{b['id']}']") or next
+        ins << b.dup
+        m << b["id"]
+      end
+    end
+
+    def insert_indirect_biblio(ins, refs, prefix)
+      refs.each do |x|
+        ins << <<~BIBENTRY
+          <bibitem id="#{x}" type="internal">
+          <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+          </bibitem>
+        BIBENTRY
+      end
+    end
+  end
+end