metanorma 1.3.5 → 1.3.9

data/lib/metanorma/collection_renderer.rb

@@ -35,12 +35,14 @@ module Metanorma
 
       @outdir = options[:output_folder]
       @coverpage = options[:coverpage]
-      @format = options[:format]
+      @format = Util.sort_extensions_execution(options[:format])
       @compile_options = options[:compile] || {}
+      @compile_options[:no_install_fonts] = true if options[:no_install_fonts]
       @log = options[:log]
       @documents = collection.documents
       @directives = collection.directives
       @disambig = Util::DisambigFiles.new
+      @compile = Compile.new
 
       # list of files in the collection
       @files = read_files folder
@@ -52,14 +54,20 @@ module Metanorma
     # @param col [Metanorma::Collection] XML collection
     # @param options [Hash]
     # @option options [String] :coverpage cover page HTML (Liquid template)
-    # @option options [Array<Synbol>] :format list of formats
+    # @option options [Array<Symbol>] :format list of formats
     # @option options [Strong] :ourput_folder output directory
     def self.render(col, options = {})
       folder = File.dirname col.file
+      # require "byebug"; byebug
+      warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
       cr = new(col, folder, options)
+      warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"
       cr.files
+      warn "\n\n\n\n\nConcatenate: #{DateTime.now.strftime('%H:%M:%S')}"
       cr.concatenate(col, options)
+      warn "\n\n\n\n\nCoverpage: #{DateTime.now.strftime('%H:%M:%S')}"
       cr.coverpage if options[:format]&.include?(:html)
+      warn "\n\n\n\n\nDone: #{DateTime.now.strftime('%H:%M:%S')}"
     end
 
     def concatenate(col, options)
@@ -70,13 +78,15 @@ module Metanorma
         ext = e == :presentation ? "presentation.xml" : e.to_s
         out = col.clone
         out.directives << "documents-inline"
-        out.documents.keys.each do |id|
-          next if @files[id][:attachment]
+        out.documents.each_key do |id|
+          next if @files[id][:attachment] || @files[id][:outputs].nil?
 
           filename = @files[id][:outputs][e]
           out.documents[id] = Metanorma::Document.raw_file(filename)
         end
-        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") { |f| f.write(out.to_xml) }
+        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") do |f|
+          f.write(out.to_xml)
+        end
       end
       options[:format].include?(:pdf) and
         pdfconv.convert(File.join(@outdir, "collection.presentation.xml"))
@@ -169,7 +179,7 @@ module Metanorma
     # @param elm [Nokogiri::XML::Element]
     # @param builder [Nokogiri::XML::Builder]
     def indexfile_docref(elm, builder)
-      return "" unless elm.at(ns("./docref"))
+      return "" unless elm.at(ns("./docref[@index = 'true']"))
 
       builder.ul { |b| docrefs(elm, b) }
     end
@@ -177,7 +187,7 @@ module Metanorma
     # @param elm [Nokogiri::XML::Element]
     # @param builder [Nokogiri::XML::Builder]
     def docrefs(elm, builder)
-      elm.xpath(ns("./docref")).each do |d|
+      elm.xpath(ns("./docref[@index = 'true']")).each do |d|
         ident = d.at(ns("./identifier")).children.to_xml
         builder.li do |li|
           li.a **{ href: index_link(d, ident) } do |a|
@@ -216,6 +226,14 @@ module Metanorma
 
     private
 
+    def format_sort(formats)
+      ret = []
+      formats.include?(:xml) and ret << :xml
+      formats.include?(:presentation) and ret << :presentation
+      a = %i(presentation xml)
+      ret + formats.reject { |i| a.include? i }
+    end
+
     # @param options [Hash]
     # @raise [ArgumentError]
     def check_options(options)

data/lib/metanorma/compile.rb

@@ -6,6 +6,8 @@ require "fontist"
 require "fontist/manifest/install"
 require_relative "compile_validate"
 require_relative "fontist_utils"
+require_relative "util"
+require_relative "sectionsplit"
 
 module Metanorma
   class Compile
@@ -15,6 +17,8 @@ module Metanorma
     def initialize
       @registry = Metanorma::Registry.instance
       @errors = []
+      @isodoc = IsoDoc::Convert.new({})
+      @fontist_installed = false
     end
 
     def compile(filename, options = {})
@@ -26,7 +30,8 @@ module Metanorma
       (file, isodoc = process_input(filename, options)) or return nil
       relaton_export(isodoc, options)
       extract(isodoc, options[:extract], options[:extract_type])
-      FontistUtils.install_fonts(@processor, options)
+      FontistUtils.install_fonts(@processor, options) unless @fontist_installed
+      @fontist_installed = true
       process_extensions(extensions, file, isodoc, options)
     end
 
@@ -35,7 +40,7 @@ module Metanorma
     end
 
     def xml_options_extract(file)
-      xml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
       if xml.root
         @registry.root_tags.each do |k, v|
           return { type: k } if v == xml.root.name
@@ -47,13 +52,13 @@ module Metanorma
     def options_extract(filename, options)
       content = read_file(filename)
       o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
-      o = o.merge(xml_options_extract(content))
+        .merge(xml_options_extract(content))
       options[:type] ||= o[:type]&.to_sym
       t = @registry.alias(options[:type]) and options[:type] = t
       dir = filename.sub(%r(/[^/]+$), "/")
       options[:relaton] ||= "#{dir}/#{o[:relaton]}" if o[:relaton]
       options[:sourcecode] ||= "#{dir}/#{o[:sourcecode]}" if o[:sourcecode]
-      options[:extension_keys] ||= o[:extensions]&.split(/,[ ]*/)&.map(&:to_sym)
+      options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
       options[:extension_keys] = nil if options[:extension_keys] == [:all]
       options[:format] ||= :asciidoc
       options[:filename] = filename
@@ -61,12 +66,10 @@ module Metanorma
     end
 
     def get_extensions(options)
-      options[:extension_keys] ||= @processor.output_formats.reduce([]) do |memo, (k, _)|
-        memo << k
-      end
+      options[:extension_keys] ||=
+        @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
       extensions = options[:extension_keys].reduce([]) do |memo, e|
-        if @processor.output_formats[e]
-          memo << e
+        if @processor.output_formats[e] then memo << e
         else
           message = "[metanorma] Error: #{e} format is not supported for this standard."
           @errors << message
@@ -84,26 +87,33 @@ module Metanorma
 
     def process_input(filename, options)
       case extname = File.extname(filename)
-      when ".adoc"
-        Util.log("[metanorma] Processing: AsciiDoc input.", :info)
-        file = read_file(filename)
-        options[:asciimath] and
-          file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
-        dir = File.dirname(filename)
-        dir != "." and
-          file.gsub!(/^include::/, "include::#{dir}/")
-        [file, @processor.input_to_isodoc(file, filename, options)]
-      when ".xml"
-        Util.log("[metanorma] Processing: Metanorma XML input.", :info)
-        # TODO NN: this is a hack -- we should provide/bridge the
-        # document attributes in Metanorma XML
-        ["", read_file(filename)]
+      when ".adoc" then process_input_adoc(filename, options)
+      when ".xml" then process_input_xml(filename, options)
       else
-        Util.log("[metanorma] Error: file extension #{extname} is not supported.", :error)
+        Util.log("[metanorma] Error: file extension #{extname} "\
+                 "is not supported.", :error)
         nil
       end
     end
 
+    def process_input_adoc(filename, options)
+      Util.log("[metanorma] Processing: AsciiDoc input.", :info)
+      file = read_file(filename)
+      options[:asciimath] and
+        file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
+      dir = File.dirname(filename)
+      dir != "." and
+        file.gsub!(/^include::/, "include::#{dir}/")
+      [file, @processor.input_to_isodoc(file, filename, options)]
+    end
+
+    def process_input_xml(filename, _options)
+      Util.log("[metanorma] Processing: Metanorma XML input.", :info)
+      # TODO NN: this is a hack -- we should provide/bridge the
+      # document attributes in Metanorma XML
+      ["", read_file(filename)]
+    end
+
     def read_file(filename)
       File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
     end
@@ -111,7 +121,7 @@ module Metanorma
     def relaton_export(isodoc, options)
       return unless options[:relaton]
 
-      xml = Nokogiri::XML(isodoc)
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
       # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
       # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
@@ -120,9 +130,7 @@ module Metanorma
 
     def clean_sourcecode(xml)
       xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
-                ".//xmlns:annotation").each do |x|
-        x.remove
-      end
+                ".//xmlns:annotation").each(&:remove)
       xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
       HTMLEntities.new.decode(xml.children.to_xml)
     end
@@ -131,11 +139,11 @@ module Metanorma
       return unless dirname
 
       if extract_types.nil? || extract_types.empty?
-        extract_types = [:sourcecode, :image, :requirement]
+        extract_types = %i[sourcecode image requirement]
       end
       FileUtils.rm_rf dirname
       FileUtils.mkdir_p dirname
-      xml = Nokogiri::XML(isodoc)
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
       sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
       image_export(xml, dirname) if extract_types.include? :image
       requirement_export(xml, dirname) if extract_types.include? :requirement
@@ -156,7 +164,7 @@ module Metanorma
       xml.at("//image | //xmlns:image") or return
       FileUtils.mkdir_p "#{dirname}/image"
       xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
-        next unless /^data:image/.match s["src"]
+        next unless /^data:image/.match? s["src"]
 
         %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
         filename = s["filename"] || sprintf("image-%04d.%s", i, imgtype)
@@ -167,8 +175,8 @@ module Metanorma
     end
 
     REQUIREMENT_XPATH = "//requirement | //xmlns:requirement | "\
-      "//recommendation | //xmlns:recommendation | //permission | "\
-      "//xmlns:permission".freeze
+                        "//recommendation | //xmlns:recommendation | //permission | "\
+                        "//xmlns:permission".freeze
 
     def requirement_export(xml, dirname)
       xml.at(REQUIREMENT_XPATH) or return
@@ -181,17 +189,6 @@ module Metanorma
       end
     end
 
-    # dependency ordering
-    def sort_extensions_execution(ext)
-      case ext
-      when :xml then 0
-      when :rxl then 1
-      when :presentation then 2
-      else
-        99
-      end
-    end
-
     def wrap_html(options, file_extension, outfilename)
       if options[:wrapper] && /html$/.match(file_extension)
         outfilename = outfilename.sub(/\.html$/, "")
@@ -206,37 +203,27 @@ module Metanorma
       f = change_output_dir options
       xml_name = f.sub(/\.[^.]+$/, ".xml")
       presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
-      extensions.sort do |a, b|
-        sort_extensions_execution(a) <=> sort_extensions_execution(b)
-      end.each do |ext|
-        isodoc_options = @processor.extract_options(file)
-        isodoc_options[:datauriimage] = true if options[:datauriimage]
-        isodoc_options[:sourcefilename] = options[:filename]
-        isodoc_options[:bare] = options[:bare]
+      Util.sort_extensions_execution(extensions).each do |ext|
         file_extension = @processor.output_formats[ext]
         outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        if ext == :pdf
-          font_locations = FontistUtils.fontist_font_locations(@processor, options)
-          font_locations and
-            isodoc_options[:mn2pdf] = { font_manifest_file: font_locations.path }
-        end
+        isodoc_options = get_isodoc_options(file, options, ext)
         if ext == :rxl
-          options[:relaton] = outfilename
-          relaton_export(isodoc, options)
+          relaton_export(isodoc, options.merge(relaton: outfilename))
         elsif options[:passthrough_presentation_xml] && ext == :presentation
           FileUtils.cp f, presentationxml_name
+        elsif ext == :html && options[:sectionsplit]
+          sectionsplit_convert(xml_name, isodoc, outfilename, isodoc_options)
         else
           begin
-            @processor.use_presentation_xml(ext) ?
-              @processor.output(nil, presentationxml_name, outfilename, ext, isodoc_options) :
-              @processor.output(isodoc, xml_name, outfilename, ext, isodoc_options)
-          rescue StandardError => e
-            if e.message.include? "Fatal:"
-              @errors << e.message
+            if @processor.use_presentation_xml(ext)
+              @processor.output(nil, presentationxml_name, outfilename, ext,
+                                isodoc_options)
             else
-              puts e.message
-              puts e.backtrace.join("\n")
+              @processor.output(isodoc, xml_name, outfilename, ext,
+                                isodoc_options)
             end
+          rescue StandardError => e
+            isodoc_error_process(e)
           end
         end
         wrap_html(options, file_extension, outfilename)
@@ -245,6 +232,29 @@ module Metanorma
 
     private
 
+    def isodoc_error_process(err)
+      if err.message.include? "Fatal:"
+        @errors << err.message
+      else
+        puts err.message
+        puts err.backtrace.join("\n")
+      end
+    end
+
+    def get_isodoc_options(file, options, ext)
+      isodoc_options = @processor.extract_options(file)
+      isodoc_options[:datauriimage] = true if options[:datauriimage]
+      isodoc_options[:sourcefilename] = options[:filename]
+      %i(bare sectionsplit no_install_fonts).each do |x|
+        isodoc_options[x] ||= options[x]
+      end
+      if ext == :pdf
+        floc = FontistUtils.fontist_font_locations(@processor, options) and
+          isodoc_options[:mn2pdf] = { font_manifest_file: floc.path }
+      end
+      isodoc_options
+    end
+
     # @param options [Hash]
     # @return [String]
     def change_output_dir(options)

data/lib/metanorma/document.rb

@@ -1,13 +1,15 @@
 module Metanorma
   class Document
     # @return [Strin]
-    attr_reader :file, :attachment, :bibitem
+    attr_reader :file, :attachment, :bibitem, :index
 
     # @param bibitem [RelatonBib::BibliographicItem]
     def initialize(bibitem, file, options = {})
       @bibitem = bibitem
       @file = file
       @attachment = options[:attachment]
+      @index = options[:index]
+      @index = true if @index.nil?
       @raw = options[:raw]
     end
 
@@ -16,10 +18,11 @@ module Metanorma
       # @param attachment [Bool] is an attachment
       # @param identifier [String] is the identifier assigned the file
       # in the collection file
+      # @param index [Bool] is indication on whether to index this file in coverpage
       # @return [Metanorma::Document]
-      def parse_file(file, attachment, identifier = nil)
+      def parse_file(file, attachment, identifier = nil, index = true)
         new(bibitem(file, attachment, identifier), file,
-            { attachment: attachment })
+            { attachment: attachment, index: index })
       end
 
       # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
@@ -30,7 +33,9 @@ module Metanorma
 
       # raw XML file, can be used to put in entire file instead of just bibitem
       def raw_file(filename)
-        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8"))
+        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8")) do |config|
+          config.huge
+        end
         new(doc, filename, raw: true)
       end
 
@@ -100,7 +105,9 @@ module Metanorma
       if @raw
         builder << @bibitem.root.to_xml
       else
-        builder.send(type + "-standard") { |b| b << @bibitem.to_xml(bibdata: true) }
+        builder.send("#{type}-standard") do |b|
+          b << @bibitem.to_xml(bibdata: true)
+        end
       end
     end
   end

data/lib/metanorma/input/asciidoc.rb

@@ -57,9 +57,9 @@ module Metanorma
       end
 
       ADOC_OPTIONS = %w(htmlstylesheet htmlcoverpage htmlintropage scripts
-                        scripts-override scripts-pdf wordstylesheet bare i18nyaml
+                        scripts-override scripts-pdf wordstylesheet i18nyaml
                         standardstylesheet header wordcoverpage wordintropage
-                        ulstyle olstyle htmlstylesheet-override
+                        ulstyle olstyle htmlstylesheet-override bare
                         htmltoclevels doctoclevels sectionsplit
                         body-font header-font monospace-font title-font
                         wordstylesheet-override).freeze

data/lib/metanorma/processor.rb

@@ -3,10 +3,7 @@
 
 module Metanorma
   class Processor
-
-    attr_reader :short
-    attr_reader :input_format
-    attr_reader :asciidoctor_backend
+    attr_reader :short, :input_format, :asciidoctor_backend
 
     def initialize
       raise "This is an abstract class!"
@@ -16,7 +13,7 @@ module Metanorma
       {
         xml: "xml",
         presentation: "presentation.xml",
-        rxl: "rxl"
+        rxl: "rxl",
       }
     end
 
@@ -36,7 +33,7 @@ module Metanorma
       end
     end
 
-    def output(isodoc_node, inname, outname, format, options={})
+    def output(isodoc_node, _inname, outname, _format, _options = {})
       File.open(outname, "w:UTF-8") { |f| f.write(isodoc_node) }
     end
 

data/lib/metanorma/sectionsplit.rb

@@ -0,0 +1,254 @@
+require "yaml"
+
+module Metanorma
+  class Compile
+    # assume we pass in Presentation XML, but we want to recover Semantic XML
+    def sectionsplit_convert(input_filename, file, output_filename = nil,
+                             opts = {})
+      @isodoc = IsoDoc::Convert.new({})
+      input_filename += ".xml" unless input_filename.match?(/\.xml$/)
+      File.exist?(input_filename) or
+        File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
+      presxml = File.read(input_filename, encoding: "utf-8")
+      @openmathdelim, @closemathdelim = @isodoc.extract_delims(presxml)
+      xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
+      build_collection(xml, presxml, output_filename || filename, dir, opts)
+    end
+
+    def ns(xpath)
+      @isodoc.ns(xpath)
+    end
+
+    def build_collection(xml, presxml, filename, dir, opts = {})
+      base = File.basename(filename)
+      collection_setup(base, dir)
+      files = sectionsplit(xml, base, dir)
+      collection_manifest(base, files, xml, presxml, dir).render(
+        { format: %i(html), output_folder: "#{filename}_collection",
+          coverpage: File.join(dir, "cover.html") }.merge(opts),
+      )
+    end
+
+    def collection_manifest(filename, files, origxml, _presxml, dir)
+      File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
+        f.write(collectionyaml(files, origxml))
+      end
+      Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
+    end
+
+    def collection_setup(filename, dir)
+      FileUtils.mkdir_p "#{filename}_collection" if filename
+      FileUtils.mkdir_p dir
+      File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
+        f.write(coll_cover)
+      end
+    end
+
+    def coll_cover
+      <<~COVER
+        <html><head/>
+            <body>
+              <h1>{{ doctitle }}</h1>
+              <h2>{{ docnumber }}</h2>
+              <nav>{{ labels["navigation"] }}</nav>
+            </body>
+        </html>
+      COVER
+    end
+
+    SPLITSECTIONS =
+      [["//preface/*", "preface"], ["//sections/*", "sections"],
+       ["//annex", nil],
+       ["//bibliography/*[not(@hidden = 'true')]", "bibliography"],
+       ["//indexsect", nil]].freeze
+
+    def sectionsplit(xml, filename, dir)
+      @key = xref_preprocess(xml)
+      @splitdir = dir
+      out = emptydoc(xml)
+      SPLITSECTIONS.each_with_object([]) do |n, ret|
+        xml.xpath(ns(n[0])).each do |s|
+          ret << sectionfile(xml, out, "#{filename}.#{ret.size}", s, n[1])
+        end
+      end
+    end
+
+    def emptydoc(xml)
+      out = xml.dup
+      out.xpath(
+        ns("//preface | //sections | //annex | //bibliography/clause | "\
+           "//bibliography/references[not(@hidden = 'true')] | //indexsect"),
+      ).each(&:remove)
+      out
+    end
+
+    def sectionfile(fulldoc, xml, file, chunk, parentnode)
+      fname = create_sectionfile(fulldoc, xml.dup, file, chunk, parentnode)
+      { order: chunk["displayorder"].to_i, url: fname,
+        title: titlerender(chunk) }
+    end
+
+    def create_sectionfile(xml, out, file, chunk, parentnode)
+      ins = out.at(ns("//misccontainer")) || out.at(ns("//bibdata"))
+      if parentnode
+        ins.next = "<#{parentnode}/>"
+        ins.next.add_child(chunk.dup)
+      else ins.next = chunk.dup
+      end
+      xref_process(out, xml, @key)
+      outname = "#{file}.xml"
+      File.open(File.join(@splitdir, outname), "w:UTF-8") { |f| f.write(out) }
+      outname
+    end
+
+    def xref_preprocess(xml)
+      svg_preprocess(xml)
+      key = (0...8).map { rand(65..90).chr }.join # random string
+      xml.root["type"] = key # to force recognition of internal refs
+      key
+    end
+
+    def xref_process(section, xml, key)
+      refs = eref_to_internal_eref(section, xml, key)
+      refs += xref_to_internal_eref(section, key)
+      ins = new_hidden_ref(section)
+      copied_refs = copy_repo_items_biblio(ins, section, xml)
+      insert_indirect_biblio(ins, refs - copied_refs, key)
+    end
+
+    def svg_preprocess(xml)
+      xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
+        m = svgmap_wrap(s)
+        s.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
+          next unless /^#/.match? a["href"]
+
+          a["href"] = a["href"].sub(/^#/, "")
+          m << "<target href='#{a['href']}'>"\
+               "<xref target='#{a['href']}'/></target>"
+        end
+      end
+    end
+
+    def svgmap_wrap(svg)
+      ret = svg.at("./ancestor::xmlns:svgmap") and return ret
+      ret = svg.at("./ancestor::xmlns:figure")
+      ret.wrap("<svgmap/>")
+      svg.at("./ancestor::xmlns:svgmap")
+    end
+
+    def make_anchor(anchor)
+      "<localityStack><locality type='anchor'><referenceFrom>"\
+        "#{anchor}</referenceFrom></locality></localityStack>"
+    end
+
+    def xref_to_internal_eref(xml, key)
+      xml.xpath(ns("//xref")).each_with_object({}) do |x, m|
+        x["bibitemid"] = "#{key}_#{x['target']}"
+        x << make_anchor(x["target"])
+        m[x["bibitemid"]] = true
+        x.delete("target")
+        x["type"] = key
+        x.name = "eref"
+      end.keys
+    end
+
+    def eref_to_internal_eref(section, xml, key)
+      eref_to_internal_eref_select(section, xml).each_with_object([]) do |x, m|
+        url = xml.at(ns("//bibitem[@id = '#{x}']/url[@type = 'citation']"))
+        section.xpath(("//*[@bibitemid = '#{x}']")).each do |e|
+          id = eref_to_internal_eref1(e, key, url)
+          id and m << id
+        end
+      end
+    end
+
+    def eref_to_internal_eref1(elem, key, url)
+      if url
+        elem.name = "link"
+        elem["target"] = url
+        nil
+      else
+        elem["bibitemid"] = "#{key}_#{elem['bibitemid']}"
+        elem << make_anchor(elem["bibitemid"])
+        elem["type"] = key
+        elem["bibitemid"]
+      end
+    end
+
+    def eref_to_internal_eref_select(section, xml)
+      refs = section.xpath(("//*/@bibitemid")).map { |x| x.text } # rubocop:disable Style/SymbolProc
+      refs.uniq.reject do |x|
+        xml.at(ns("//bibitem[@id = '#{x}'][@type = 'internal']")) ||
+          xml.at(ns("//bibitem[@id = '#{x}']"\
+                    "[docidentifier/@type = 'repository']"))
+      end
+    end
+
+    # from standoc
+    def new_hidden_ref(xmldoc)
+      ins = xmldoc.at("bibliography") or
+        xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+      ins.add_child("<references hidden='true' normative='false'/>").first
+    end
+
+    def copy_repo_items_biblio(ins, section, xml)
+      xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
+        .each_with_object([]) do |b, m|
+        section.at("//*[@bibitemid = '#{b['id']}']") or next
+        ins << b.dup
+        m << b["id"]
+      end
+    end
+
+    def insert_indirect_biblio(ins, refs, prefix)
+      refs.each do |x|
+        ins << <<~BIBENTRY
+          <bibitem id="#{x}" type="internal">
+          <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+          </bibitem>
+        BIBENTRY
+      end
+    end
+
+    def recursive_string_keys(hash)
+      case hash
+      when Hash then hash.map { |k, v| [k.to_s, recursive_string_keys(v)] }.to_h
+      when Enumerable then hash.map { |v| recursive_string_keys(v) }
+      else
+        hash
+      end
+    end
+
+    def titlerender(section)
+      title = section.at(ns("./title")) or return "[Untitled]"
+      t = title.dup
+      t.xpath(ns(".//tab | .//br")).each { |x| x.replace(" ") }
+      t.xpath(ns(".//strong")).each { |x| x.replace(x.children) }
+      t.children.to_xml
+    end
+
+    def collectionyaml(files, xml)
+      ret = {
+        directives: ["presentation-xml", "bare-after-first"],
+        bibdata: {
+          title: {
+            type: "title-main", language: @lang,
+            content: xml.at(ns("//bibdata/title")).text
+          },
+          type: "collection",
+          docid: {
+            type: xml.at(ns("//bibdata/docidentifier/@type")).text,
+            id: xml.at(ns("//bibdata/docidentifier")).text,
+          },
+        },
+        manifest: {
+          level: "collection", title: "Collection",
+          docref: files.sort_by { |f| f[:order] }.each.map do |f|
+            { fileref: f[:url], identifier: f[:title] }
+          end
+        },
+      }
+      recursive_string_keys(ret).to_yaml
+    end
+  end
+end