metanorma 1.7.6 → 2.0.0

data/lib/metanorma/collection_render_utils.rb

@@ -1,169 +0,0 @@
-module Metanorma
-  class CollectionRenderer
-    def dir_name_cleanse(name)
-      path = Pathname.new(name)
-      clean_regex = /[<>:"|?*]/
-      fallback_sym = "_"
-      return name.gsub(clean_regex, fallback_sym) unless path.absolute?
-
-      File.join(path.dirname,
-                path.basename.to_s.gsub(clean_regex, fallback_sym))
-    end
-
-    def dup_bibitem(docid, bib)
-      newbib = @files.get(docid, :bibdata).dup
-      newbib.name = "bibitem"
-      newbib["hidden"] = "true"
-      newbib&.at("./*[local-name() = 'ext']")&.remove
-      newbib["id"] = bib["id"]
-      newbib
-    end
-
-    def get_bibitem_docid(bib, identifier)
-      docid =
-        bib.at(ns("./docidentifier[@type = 'metanorma-collection']")) ||
-        bib.at(ns("./docidentifier[not(@type)]")) ||
-        bib.at(ns("./docidentifier"))
-      docid &&= docid_prefix(docid)
-      if @files.get(docid) then docid
-      else
-        fail_update_bibitem(docid, identifier)
-        nil
-      end
-    end
-
-    def hide_refs(docxml)
-      docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or " \
-                      "@hidden = 'false'])]")).each do |f|
-        f["hidden"] = "true"
-      end
-    end
-
-    def strip_eref(eref)
-      eref.xpath(ns("./locality | ./localityStack")).each(&:remove)
-      eref.replace(eref.children)
-    end
-
-    def docid_to_citeas(bib)
-      docid = bib.at(ns("./docidentifier[@primary = 'true']")) ||
-        bib.at(ns("./docidentifier")) or return
-      docid_prefix(docid)
-    end
-
-    def collect_erefs(docxml)
-      docxml.xpath(ns("//eref"))
-        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
-        m[:citeas][i["citeas"]] = true
-        m[:bibitemid][i["bibitemid"]] = true
-      end
-    end
-
-    private
-
-    def docid_prefix(docid)
-      type = docid["type"]
-      type == "metanorma-collection" and type = nil
-      @c.decode(@isodoc
-          .docid_prefix(type, docid.children.to_xml)).gsub(/\s/, " ")
-    end
-
-    def create_non_existing_directory(output_directory)
-      !File.exist?(output_directory) and
-        FileUtils.mkdir_p(output_directory)
-    end
-
-    def format_sort(formats)
-      ret = []
-      formats.include?(:xml) and ret << :xml
-      formats.include?(:presentation) and ret << :presentation
-      a = %i(presentation xml)
-      ret + formats.reject { |i| a.include? i }
-    end
-
-    # @param options [Hash]
-    # @raise [ArgumentError]
-    def check_options(options)
-      (options[:format].is_a?(Array) && (FORMATS & options[:format]).any?) or
-        raise ArgumentError, "Need to specify formats (xml,html,pdf,doc)"
-    end
-
-    def pdfconv
-      doctype = @doctype.to_sym
-      x = Asciidoctor.load nil, backend: doctype
-      x.converter.pdf_converter(PdfOptionsNode.new(doctype,
-                                                   @compile_options))
-    end
-
-    def fail_update_bibitem(docid, identifier)
-      error = "[metanorma] Cannot find crossreference to document #{docid} " \
-              "in document #{identifier}."
-      @log&.add("Cross-References", nil, error)
-      Util.log(error, :warning)
-    end
-
-    def datauri_encode(docxml)
-      docxml.xpath(ns("//image")).each do |i|
-        i["src"] = Vectory::Utils::datauri(i["src"])
-      end
-      docxml
-    end
-
-    class PdfOptionsNode
-      def initialize(doctype, options)
-        docproc = Metanorma::Registry.instance.find_processor(doctype)
-        if FontistUtils.has_custom_fonts?(docproc, options, {})
-          @fonts_manifest = FontistUtils.location_manifest(docproc, options)
-        end
-      end
-
-      def attr(key)
-        if key == "fonts-manifest" && @fonts_manifest
-          @fonts_manifest
-        end
-      end
-    end
-
-    class Dummy
-      def attr(_key); end
-    end
-
-    def isodoc_create
-      x = Asciidoctor.load nil, backend: @doctype.to_sym
-      isodoc = x.converter.html_converter(Dummy.new) # to obtain Isodoc class
-      isodoc.i18n_init(@lang, @script, @locale) # read in internationalisation
-      isodoc.metadata_init(@lang, @script, @locale, isodoc.i18n)
-      isodoc.info(@xml, nil)
-      isodoc
-    end
-
-    # create the @meta class of isodoc, for populating Liquid,
-    # with "navigation" set to the index bar.
-    # extracted from the manifest
-    def isodoc_populate
-      @isodoc.info(@xml, nil)
-      m = @xml.at(ns("//manifest"))
-      { navigation: indexfile(m), nav_object: index_object(m),
-        docrefs: liquid_docrefs,
-        "prefatory-content": isodoc_builder(@xml.at(ns("//prefatory-content"))),
-        "final-content": isodoc_builder(@xml.at(ns("//final-content"))),
-        doctitle: m.at(ns("../bibdata/title"))&.text,
-        docnumber: m.at(ns("../bibdata/docidentifier"))&.text }.each do |k, v|
-          v and @isodoc.meta.set(
-            k, v
-          )
-        end
-    end
-
-    def isodoc_builder(node)
-      Nokogiri::HTML::Builder.new(encoding: "UTF-8") do |b|
-        b.div do |div|
-          node&.children&.each { |n| @isodoc.parse(n, div) }
-        end
-      end.doc.root.to_html
-    end
-
-    def ns(xpath)
-      @isodoc.ns(xpath)
-    end
-  end
-end

data/lib/metanorma/collection_render_word.rb

@@ -1,131 +0,0 @@
-module Metanorma
-  class CollectionRenderer
-    def docconv
-      @tempfile_cache ||= []
-      doctype = @doctype.to_sym
-      x = Asciidoctor.load nil, backend: doctype
-      x.converter.doc_converter(DocOptionsNode.new(@directives, @dirname))
-    end
-
-    def concat_extract_files(filename)
-      xml = Nokogiri::XML(File.read(filename, encoding: "UTF-8"), &:huge)
-      docs = xml.xpath(ns("//doc-container")).each_with_object([]) do |x, m|
-        n = Nokogiri::XML::Document.new
-        n.add_child(x.elements.first.remove)
-        m << n
-      end
-      [wrapping_doc(docs.first.dup, xml), docs]
-    end
-
-    def wrapping_doc(doc, xml)
-      doc.at(ns("//bibdata")).replace(xml.at(ns("//bibdata")).to_xml)
-      sections = wrapping_doc_body(doc)
-      wrapping_doc_intro_outro(xml, sections)
-      set_displayorder_wrapping_doc(doc)
-    end
-
-    def wrapping_doc_intro_outro(xml, sections)
-      p = xml.at(ns("//prefatory-content")) and
-        sections.previous = "<preface>#{p.children.to_xml}</preface>"
-      p = xml.at(ns("//final-content")) and
-        sections.next = "<annex>#{p.children.to_xml}</annex>"
-    end
-
-    def wrapping_doc_body(doc)
-      doc.xpath(ns("//annex | //preface | //bibliography")).each(&:remove)
-      s = doc.at(ns("//sections"))
-      repl = <<~BODY
-        <sections><clause id='_collection_placeholder'><p>PLACEHOLDER</p></clause></sections>
-      BODY
-      s.replace(repl)
-      doc.at(ns("//sections"))
-    end
-
-    def set_displayorder_wrapping_doc(doc)
-      doc.xpath(ns("//preface/* | //sections/* | //annex"))
-        .each_with_index do |x, i|
-        x["displayorder"] = i + 1
-      end
-      doc
-    end
-
-    SECTION_BREAK = '<p class="MsoNormal"><br clear="all" class="section"/></p>'
-      .freeze
-    DIV1 = '<div class="WordSection1">&#xa0;</div>'.freeze
-    DIV2 = '<div class="WordSection2">&#xa0;</div>'.freeze
-
-    def docconv_convert1(docs)
-      docs.each_with_index.with_object([]) do |(d, i), m|
-        conv = docconv
-        conv.convert_init(d.to_xml(encoding: "UTF-8"), "xxxx", false)
-        html = conv.postprocess_cleanup(conv.convert1(d, "xxx", "."))
-        @tempfile_cache += conv.tempfile_cache # hold on to the temp img files
-        b = Nokogiri::XML(html).at("//body")
-        i == docs.size - 1 or
-          b << '<p class="MsoNormal"><br clear="all" class="section"/></p>'
-        m << b.children
-      end
-    end
-
-    def collection_coverpages(conv, docs)
-      conv.wordintropage and [DIV2, SECTION_BREAK].reverse.each do |s|
-        docs.unshift(Nokogiri::XML(s).root)
-      end
-      conv.wordcoverpage and [DIV1, SECTION_BREAK].reverse.each do |s|
-        docs.unshift(Nokogiri::XML(s).root)
-      end
-      docs
-    end
-
-    def docconv_convert(filename)
-      pref_file, docs = concat_extract_files(filename)
-      body = docconv_convert1(docs)
-      collection_conv = overall_docconv_converter(body)
-      collection_coverpages(collection_conv, body)
-      collection_conv.convert(filename, pref_file.to_xml, false)
-    end
-
-    def overall_docconv_cover(collection_conv)
-      p = Util::hash_key_detect(@directives, "collection-word-coverpage", nil)
-      collection_conv.wordcoverpage =
-        Util::rel_path_resolve(@dirname, p)
-      p = Util::hash_key_detect(@directives, "collection-word-intropage", nil)
-      collection_conv.wordintropage =
-        Util::rel_path_resolve(@dirname, p)
-    end
-
-    def overall_docconv_converter(body)
-      collection_conv = docconv
-      collection_conv.options[:collection_doc] = body.map(&:to_xml).join
-      overall_docconv_cover(collection_conv)
-
-      def collection_conv.postprocess_cleanup(result)
-        ret = to_xhtml(super)
-        b = ret.at("//div[@id = '_collection_placeholder']")
-        b.replace(@options[:collection_doc])
-        from_xhtml(ret)
-      end
-
-      collection_conv
-    end
-
-    class DocOptionsNode
-      def initialize(directives, dir)
-        @dir = dir
-        @wordcoverpage =
-          Util::hash_key_detect(directives, "document-word-coverpage",
-                                @wordcoverpage)
-        @wordintropage =
-          Util::hash_key_detect(directives, "document-word-intropage",
-                                @wordintropage)
-      end
-
-      def attr(key)
-        case key
-        when "wordcoverpage" then Util::rel_path_resolve(@dir, @wordcoverpage)
-        when "wordintropage" then Util::rel_path_resolve(@dir, @wordintropage)
-        end
-      end
-    end
-  end
-end

data/lib/metanorma/collection_renderer.rb

@@ -1,230 +0,0 @@
-require "isodoc"
-require "htmlentities"
-require_relative "collection_fileprocess"
-require_relative "fontist_utils"
-require_relative "util"
-require_relative "files_lookup"
-require_relative "collection_render_utils"
-require_relative "collection_render_word"
-
-module Metanorma
-  # XML collection renderer
-  class CollectionRenderer
-    FORMATS = %i[html xml doc pdf].freeze
-
-    attr_accessor :isodoc, :nested
-    attr_reader :xml, :compile, :compile_options, :documents
-
-    # This is only going to render the HTML collection
-    # @param xml [Metanorma::Collection] input XML collection
-    # @param folder [String] input folder
-    # @param options [Hash]
-    # @option options [String] :coverpage cover page HTML (Liquid template)
-    # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
-    # @option options [String] :output_folder output directory
-    #
-    # We presuppose that the bibdata of the document is equivalent to that of
-    # the collection, and that the flavour gem can sensibly process it. We may
-    # need to enhance metadata in the flavour gems isodoc/metadata.rb with
-    # collection metadata
-    def initialize(collection, folder, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      check_options options
-      @xml = Nokogiri::XML collection.to_xml # @xml is the collection manifest
-      @lang = @xml.at("//xmlns:bibdata/xmlns:language")&.text || "en"
-      @script = @xml.at("//xmlns:bibdata/xmlns:script")&.text || "Latn"
-      @locale = @xml.at("//xmlns:bibdata/xmlns:locale")&.text
-      @doctype = doctype
-      @compile = Compile.new
-      @compile.load_flavor(@doctype)
-
-      @isodoc = isodoc_create # output processor for flavour
-      @outdir = dir_name_cleanse(options[:output_folder])
-      @coverpage = options[:coverpage] || collection.coverpage
-      @format = Util.sort_extensions_execution(options[:format])
-      @compile_options = options[:compile] || {}
-      @compile_options[:no_install_fonts] = true if options[:no_install_fonts]
-      @log = options[:log]
-      @documents = collection.documents
-      @bibdata = collection.documents
-      @directives = collection.directives
-      @dirname = collection.dirname
-      @disambig = Util::DisambigFiles.new
-      @c = HTMLEntities.new
-      @files_to_delete = []
-      @nested = options[:nested] # if false, this is the root instance of Renderer
-      # if true, then this is not the last time Renderer will be run
-      # (e.g. this is sectionsplit)
-
-      # list of files in the collection
-      @files = Metanorma::FileLookup.new(folder, self)
-      @files.add_section_split
-      isodoc_populate
-      create_non_existing_directory(@outdir)
-    end
-
-    def flush_files
-      warn "\n\n\n\n\nDone: #{DateTime.now.strftime('%H:%M:%S')}"
-      warn @files.files_to_delete
-      @files.files_to_delete.each { |f| FileUtils.rm_f(f) }
-      @files_to_delete.each { |f| FileUtils.rm_f(f) }
-    end
-
-    # @param col [Metanorma::Collection] XML collection
-    # @param options [Hash]
-    # @option options [String] :coverpage cover page HTML (Liquid template)
-    # @option options [Array<Symbol>] :format list of formats
-    # @option options [Strong] :ourput_folder output directory
-    def self.render(col, options = {})
-      warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
-      cr = new(col, File.dirname(col.file), options)
-      cr.files
-      cr.concatenate(col, options)
-      options[:format]&.include?(:html) and cr.coverpage
-      cr.flush_files
-      cr
-    end
-
-    def concatenate(col, options)
-      warn "\n\n\n\n\nConcatenate: #{DateTime.now.strftime('%H:%M:%S')}"
-      (options[:format] & %i(pdf doc)).empty? or
-        options[:format] << :presentation
-      concatenate_prep(col, options)
-      concatenate_outputs(options)
-    end
-
-    def concatenate_prep(col, options)
-      %i(xml presentation).each do |e|
-        options[:format].include?(e) or next
-        ext = e == :presentation ? "presentation.xml" : e.to_s
-        File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") do |f|
-          f.write(concatenate1(col.clone, e).to_xml)
-        end
-      end
-    end
-
-    def concatenate_outputs(options)
-      pres = File.join(@outdir, "collection.presentation.xml")
-      options[:format].include?(:pdf) and pdfconv.convert(pres)
-      options[:format].include?(:doc) and docconv_convert(pres)
-    end
-
-    def concatenate1(out, ext)
-      out.directives << "documents-inline"
-      out.bibdatas.each_key do |ident|
-        id = @isodoc.docid_prefix(nil, ident.dup)
-        @files.get(id, :attachment) || @files.get(id, :outputs).nil? and next
-        out.documents[Util::key id] =
-          Metanorma::Document.raw_file(@files.get(id, :outputs)[ext])
-      end
-      out
-    end
-
-    # infer the flavour from the first document identifier; relaton does that
-    def doctype
-      if (docid = @xml.at("//xmlns:bibdata/xmlns:docidentifier/@type")&.text)
-        dt = docid.downcase
-      elsif (docid = @xml.at("//xmlns:bibdata/xmlns:docidentifier")&.text)
-        dt = docid.sub(/\s.*$/, "").lowercase
-      else return "standoc"
-      end
-      @registry = Metanorma::Registry.instance
-      @registry.alias(dt.to_sym)&.to_s || dt
-    end
-
-    # populate liquid template of ARGV[1] with metadata extracted from
-    # collection manifest
-    def coverpage
-      @coverpage or return
-      warn "\n\n\n\n\nCoverpage: #{DateTime.now.strftime('%H:%M:%S')}"
-      File.open(File.join(@outdir, "index.html"), "w:UTF-8") do |f|
-        f.write @isodoc.populate_template(File.read(@coverpage))
-      end
-    end
-
-    # @param elm [Nokogiri::XML::Element]
-    # @return [String]
-    def indexfile_title(elm)
-      elm.at(ns("./title"))&.text
-    end
-
-    # uses the identifier to label documents; other attributes (title) can be
-    # looked up in @files[id][:bibdata]
-    #
-    # @param elm [Nokogiri::XML::Element]
-    # @param builder [Nokogiri::XML::Builder]
-    def indexfile_docref(elm, builder)
-      return "" unless elm.at(ns("./docref[@index = 'true']"))
-
-      builder.ul { |b| docrefs(elm, b) }
-    end
-
-    # @param elm [Nokogiri::XML::Element]
-    # @param builder [Nokogiri::XML::Builder]
-    def docrefs(elm, builder)
-      elm.xpath(ns("./docref[@index = 'true']")).each do |d|
-        ident = docref_ident(d)
-        builder.li do |li|
-          li.a href: index_link(d, ident) do |a|
-            a << ident.split(/([<>&])/).map do |x|
-              /[<>&]/.match?(x) ? x : @c.encode(x, :hexadecimal)
-            end.join
-          end
-        end
-      end
-    end
-
-    def docref_ident(docref)
-      ident = docref.at(ns("./identifier")).children.to_xml
-      @c.decode(@isodoc.docid_prefix(nil, ident))
-    end
-
-    def index_link(docref, ident)
-      if docref["fileref"]
-        @files.get(ident, :out_path).sub(/\.xml$/, ".html")
-      else "#{docref['id']}.html"
-      end
-    end
-
-    # single level navigation list, with hierarchical nesting
-    #
-    # @param elm [Nokogiri::XML::Element]
-    # @return [String] XML
-    def indexfile(elm)
-      Nokogiri::HTML::Builder.new do |b|
-        b.ul do
-          b.li indexfile_title(elm)
-          indexfile_docref(elm, b)
-          elm.xpath(ns("./manifest")).each do |d|
-            b << indexfile(d)
-          end
-        end
-      end.doc.root.to_html
-    end
-
-    # object to construct navigation out of in Liquid
-    def index_object(elm)
-      c = elm.xpath(ns("./manifest")).each_with_object([]) do |d, b|
-        b << index_object(d)
-      end
-      c.empty? and c = nil
-      r = Nokogiri::HTML::Builder.new do |b|
-        indexfile_docref(elm, b)
-      end
-      r &&= r.doc.root&.to_html&.gsub("\n", " ")
-      { title: indexfile_title(elm),
-        docrefs: r, children: c }.compact
-    end
-
-    def liquid_docrefs
-      @xml.xpath(ns("//docref[@index = 'true']")).each_with_object([]) do |d, m|
-        ident = d.at(ns("./identifier")).children.to_xml
-        ident = @c.decode(@isodoc.docid_prefix(nil, ident))
-        title = d.at(ns("./bibdata/title[@type = 'main']")) ||
-          d.at(ns("./bibdata/title")) || d.at(ns("./title"))
-        m << { "identifier" => ident, "file" => index_link(d, ident),
-               "title" => title&.children&.to_xml,
-               "level" => d.at(ns("./level"))&.text }
-      end
-    end
-  end
-end

data/lib/metanorma/document.rb

@@ -1,133 +0,0 @@
-module Metanorma
-  class Document
-    # @return [Strin]
-    attr_reader :file, :attachment, :bibitem, :index
-
-    # @param bibitem [RelatonBib::BibliographicItem]
-    def initialize(bibitem, file, options = {})
-      @bibitem = bibitem
-      @file = file
-      @attachment = options[:attachment]
-      @index = options[:index]
-      @index = true if @index.nil?
-      @raw = options[:raw]
-    end
-
-    class << self
-      # @param file [String] file path
-      # @param attachment [Bool] is an attachment
-      # @param identifier [String] is the identifier assigned the file
-      # in the collection file
-      # @param index [Bool] is indication on whether to index this file in coverpage
-      # @return [Metanorma::Document]
-      def parse_file(file, attachment, identifier = nil, index = true)
-        new(bibitem(file, attachment, identifier), file,
-            { attachment: attachment, index: index })
-      end
-
-      # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
-      # @return [Metanorma::Document]
-      def parse_xml(xml)
-        new from_xml(xml)
-      end
-
-      # raw XML file, can be used to put in entire file instead of just bibitem
-      def raw_file(filename)
-        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8")) do |config|
-          config.huge
-        end
-        new(doc, filename, raw: true)
-      end
-
-      def attachment_bibitem(identifier)
-        Nokogiri::XML <<~DOCUMENT
-          <bibdata><docidentifier>#{identifier}</docidentifier></bibdata>
-        DOCUMENT
-      end
-
-      private
-
-      def mn2relaton_parser(tag)
-        case tag.sub(/-standard/, "")
-        when "bipm" then RelatonBipm::XMLParser
-        when "bsi" then RelatonBsi::XMLParser
-        when "ietf" then RelatonIetf::XMLParser
-        when "iho" then RelatonIho::XMLParser
-        when "itu" then RelatonItu::XMLParser
-        when "iec" then RelatonIec::XMLParser
-        when "iso" then RelatonIsoBib::XMLParser
-        when "nist" then RelatonNist::XMLParser
-        when "ogc" then RelatonOgc::XMLParser
-        else RelatonBib::XMLParser
-        end
-      end
-
-      # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
-      # @return [RelatonBib::BibliographicItem,RelatonIso::IsoBibliographicItem]
-      def from_xml(xml)
-        b = xml.at("//xmlns:bibitem|//xmlns:bibdata")
-        r = mn2relaton_parser(xml.root.name)
-        r.from_xml(b.to_xml)
-      end
-
-      # @param file [String]
-      # @return [Symbol] file type
-      def format(file)
-        case file
-        when /\.xml$/ then :xml
-        when /.ya?ml$/ then :yaml
-        end
-      end
-
-      # @param file [String]
-      # @return [RelatonBib::BibliographicItem,
-      #   RelatonIso::IsoBibliographicItem]
-      def bibitem(file, attachment, identifier)
-        if attachment then attachment_bibitem(identifier)
-        else
-          case format(file)
-          when :xml
-            from_xml (Nokogiri::XML(File.read(file, encoding: "UTF-8")) { |x| x.huge })
-          when :yaml
-            yaml = File.read(file, encoding: "UTF-8")
-            Relaton::Cli::YAMLConvertor.convert_single_file(yaml)
-          end
-        end
-      end
-    end
-
-    # @param builder [Nokogiri::XML::Builder, nil]
-    # @return [Nokogiri::XML::Builder, String]
-    def to_xml(builder = nil)
-      if builder
-        render_xml builder
-      else
-        Nokogiri::XML::Builder.new do |b|
-          root = render_xml b
-          root["xmlns"] = "http://metanorma.org"
-        end.to_xml
-      end
-    end
-
-    # @return [String]
-    def type
-      first = @bibitem.docidentifier.first
-      @type ||= (first&.type&.downcase ||
-                 first&.id&.match(/^[^\s]+/)&.to_s)&.downcase ||
-        "standoc"
-    end
-
-    private
-
-    def render_xml(builder)
-      if @raw
-        #builder << @bibitem.root.to_xml
-        builder.parent.add_child(@bibitem.root)
-      else
-        builder.send("#{type}-standard") do |b|
-          b << @bibitem.to_xml(bibdata: true)
-        end
-      end
-    end
-  end
-end