metanorma 1.7.6 → 2.0.0

data/lib/metanorma/collection/renderer/renderer.rb

@@ -0,0 +1,157 @@
+require "isodoc"
+require "htmlentities"
+require_relative "fileprocess"
+require_relative "../../util/fontist_helper"
+require_relative "../../util/util"
+require_relative "../filelookup/filelookup"
+require_relative "utils"
+require_relative "render_word"
+require_relative "navigation"
+
+module Metanorma
+  class Collection
+    class Renderer
+      FORMATS = %i[html xml doc pdf].freeze
+
+      attr_accessor :isodoc, :nested
+      attr_reader :xml, :compile, :compile_options, :documents, :outdir,
+                  :manifest
+
+      # This is only going to render the HTML collection
+      # @param xml [Metanorma::Collection] input XML collection
+      # @param folder [String] input folder
+      # @param options [Hash]
+      # @option options [String] :coverpage cover page HTML (Liquid template)
+      # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
+      # @option options [String] :output_folder output directory
+      #
+      # We presuppose that the bibdata of the document is equivalent to that of
+      # the collection, and that the flavour gem can sensibly process it. We may
+      # need to enhance metadata in the flavour gems isodoc/metadata.rb with
+      # collection metadata
+      def initialize(collection, folder, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        check_options options
+        @xml = Nokogiri::XML collection.to_xml # @xml is the collection manifest
+        @xml.root.default_namespace = "http://metanorma.org"
+        @lang = collection.bibdata.language.first || "en"
+        @script = collection.bibdata.script.first || "Latn"
+        @locale = @xml.at("//xmlns:bibdata/xmlns:locale")&.text
+        @doctype = doctype
+        @compile = Compile.new
+        @compile.load_flavor(@doctype)
+
+        @isodoc = isodoc_create # output processor for flavour
+        @outdir = dir_name_cleanse(options[:output_folder])
+        @coverpage = options[:coverpage] || collection.coverpage
+        @format = ::Metanorma::Util.sort_extensions_execution(options[:format])
+        @compile_options = options[:compile] || {}
+        @compile_options[:install_fonts] = true if options[:install_fonts]
+        @log = options[:log]
+        @bibdata = collection.bibdata
+        @documents = collection.documents
+        @bibdatas = collection.documents
+        @directives = collection.directives
+        @dirname = collection.dirname
+        @manifest = collection.manifest.config
+        @disambig = Util::DisambigFiles.new
+        @prefatory = collection.prefatory
+        @final = collection.final
+        @c = HTMLEntities.new
+        @files_to_delete = []
+        @nested = options[:nested] # if false, this is the root instance of Renderer
+        # if true, then this is not the last time Renderer will be run
+        # (e.g. this is sectionsplit)
+
+        # list of files in the collection
+        @files = Metanorma::Collection::FileLookup.new(folder, self)
+        @files.add_section_split
+        isodoc_populate
+        create_non_existing_directory(@outdir)
+      end
+
+      def flush_files
+        warn "\n\n\n\n\nDone: #{DateTime.now.strftime('%H:%M:%S')}"
+        warn @files.files_to_delete
+        @files.files_to_delete.each { |f| FileUtils.rm_f(f) }
+        @files_to_delete.each { |f| FileUtils.rm_f(f) }
+      end
+
+      # @param col [Metanorma::Collection] XML collection
+      # @param options [Hash]
+      # @option options [String] :coverpage cover page HTML (Liquid template)
+      # @option options [Array<Symbol>] :format list of formats
+      # @option options [Strong] :ourput_folder output directory
+      def self.render(col, options = {})
+        warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
+        cr = new(col, File.dirname(col.file), options)
+        cr.files
+        cr.concatenate(col, options)
+        options[:format]&.include?(:html) and cr.coverpage
+        cr.flush_files
+        cr
+      end
+
+      def concatenate(col, options)
+        warn "\n\n\n\n\nConcatenate: #{DateTime.now.strftime('%H:%M:%S')}"
+        (options[:format] & %i(pdf doc)).empty? or
+          options[:format] << :presentation
+        concatenate_prep(col, options)
+        concatenate_outputs(options)
+      end
+
+      def concatenate_prep(col, options)
+        %i(xml presentation).each do |e|
+          options[:format].include?(e) or next
+          ext = e == :presentation ? "presentation.xml" : e.to_s
+          File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") do |f|
+            b = concatenate1(col.clone, e).to_xml
+            e == :presentation and
+              b.sub!("<metanorma-collection>", "<metanorma-collection xmlns='http://metanorma.org'>")
+            # TODO BEING FORCED TO DO THAT BECAUSE SHALE IS NOT DEALING WITH DEFAULT NAMESPACES
+            f.write(b)
+          end
+        end
+      end
+
+      def concatenate_outputs(options)
+        pres = File.join(@outdir, "collection.presentation.xml")
+        options[:format].include?(:pdf) and pdfconv.convert(pres)
+        options[:format].include?(:doc) and docconv_convert(pres)
+      end
+
+      def concatenate1(out, ext)
+        out.directives << ::Metanorma::Collection::Config::Directive.new(key: "documents-inline")
+        out.bibdatas.each_key do |ident|
+          id = @isodoc.docid_prefix(nil, ident.dup)
+          @files.get(id, :attachment) || @files.get(id, :outputs).nil? and next
+          out.documents[Util::key id] =
+            Metanorma::Collection::Document.raw_file(@files.get(id,
+                                                                :outputs)[ext])
+        end
+        out
+      end
+
+      # infer the flavour from the first document identifier; relaton does that
+      def doctype
+        if (docid = @xml.at("//bibdata/docidentifier/@type")&.text)
+          dt = docid.downcase
+        elsif (docid = @xml.at("//bibdata/docidentifier")&.text)
+          dt = docid.sub(/\s.*$/, "").lowercase
+        else return "standoc"
+        end
+        @registry = Metanorma::Registry.instance
+        @registry.alias(dt.to_sym)&.to_s || dt
+      end
+
+      # populate liquid template of ARGV[1] with metadata extracted from
+      # collection manifest
+      def coverpage
+        @coverpage or return
+        warn "\n\n\n\n\nCoverpage: #{DateTime.now.strftime('%H:%M:%S')}"
+        File.open(File.join(@outdir, "index.html"), "w:UTF-8") do |f|
+          f.write @isodoc.populate_template(File.read(@coverpage))
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/collection/renderer/utils.rb

@@ -0,0 +1,183 @@
+module Metanorma
+  class Collection
+    class Renderer
+      def dir_name_cleanse(name)
+        path = Pathname.new(name)
+        clean_regex = /[<>:"|?*]/
+        fallback_sym = "_"
+        return name.gsub(clean_regex, fallback_sym) unless path.absolute?
+
+        File.join(path.dirname,
+                  path.basename.to_s.gsub(clean_regex, fallback_sym))
+      end
+
+      def dup_bibitem(docid, bib)
+        newbib = @files.get(docid, :bibdata).dup
+        newbib.name = "bibitem"
+        newbib["hidden"] = "true"
+        newbib&.at("./*[local-name() = 'ext']")&.remove
+        newbib["id"] = bib["id"]
+        newbib
+      end
+
+      def get_bibitem_docid(bib, identifier)
+        docid =
+          bib.at(ns("./docidentifier[@type = 'metanorma-collection']")) ||
+          bib.at(ns("./docidentifier[not(@type)]")) ||
+          bib.at(ns("./docidentifier"))
+        docid &&= docid_prefix(docid)
+        if @files.get(docid) then docid
+        else
+          fail_update_bibitem(docid, identifier)
+          nil
+        end
+      end
+
+      def hide_refs(docxml)
+        docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or " \
+                        "@hidden = 'false'])]")).each do |f|
+          f["hidden"] = "true"
+        end
+      end
+
+      def strip_eref(eref)
+        eref.xpath(ns("./locality | ./localityStack")).each(&:remove)
+        eref.replace(eref.children)
+      end
+
+      def docid_to_citeas(bib)
+        docid = bib.at(ns("./docidentifier[@primary = 'true']")) ||
+          bib.at(ns("./docidentifier")) or return
+        docid_prefix(docid)
+      end
+
+      def collect_erefs(docxml)
+        docxml.xpath(ns("//eref"))
+          .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
+          m[:citeas][i["citeas"]] = true
+          m[:bibitemid][i["bibitemid"]] = true
+        end
+      end
+
+      def docid_xml(val)
+        "<docidentifier type='repository'>current-metanorma-collection/" \
+          "#{val}</docidentifier>"
+      end
+
+      def add_hidden_bibliography(xmldoc, refs)
+        ins = new_hidden_ref(xmldoc)
+        refs.each do |k, v|
+          url = @files.url(v, {})
+          ins << <<~XML
+            <bibitem id="#{k}">#{docid_xml(v)}<uri type='citation'>#{url}</uri></bibitem>
+          XML
+        end
+      end
+
+      private
+
+      def docid_prefix(docid)
+        type = docid["type"]
+        type == "metanorma-collection" and type = nil
+        @c.decode(@isodoc
+            .docid_prefix(type, docid.children.to_xml)).gsub(/\s/, " ")
+      end
+
+      def create_non_existing_directory(output_directory)
+        !File.exist?(output_directory) and
+          FileUtils.mkdir_p(output_directory)
+      end
+
+      def format_sort(formats)
+        ret = []
+        formats.include?(:xml) and ret << :xml
+        formats.include?(:presentation) and ret << :presentation
+        a = %i(presentation xml)
+        ret + formats.reject { |i| a.include? i }
+      end
+
+      # @param options [Hash]
+      # @raise [ArgumentError]
+      def check_options(options)
+        (options[:format].is_a?(Array) && (FORMATS & options[:format]).any?) or
+          raise ArgumentError, "Need to specify formats (xml,html,pdf,doc)"
+      end
+
+      def pdfconv
+        doctype = @doctype.to_sym
+        x = Asciidoctor.load nil, backend: doctype
+        x.converter.pdf_converter(PdfOptionsNode.new(doctype,
+                                                     @compile_options))
+      end
+
+      def fail_update_bibitem(docid, identifier)
+        error = "[metanorma] Cannot find crossreference to document #{docid} " \
+                "in document #{identifier}."
+        @log&.add("Cross-References", nil, error)
+        ::Metanorma::Util.log(error, :warning)
+      end
+
+      def datauri_encode(docxml, directory)
+        docxml.xpath(ns("//image")).each do |i|
+          i["src"] = Vectory::Utils::datauri(i["src"], directory)
+        end
+        docxml
+      end
+
+      class PdfOptionsNode
+        def initialize(doctype, options)
+          p = Metanorma::Registry.instance.find_processor(doctype)
+          if ::Metanorma::Util::FontistHelper.has_custom_fonts?(p, options, {})
+            @fonts_manifest =
+              ::Metanorma::Util::FontistHelper.location_manifest(p, options)
+          end
+        end
+
+        def attr(key)
+          if key == "fonts-manifest" && @fonts_manifest
+            @fonts_manifest
+          end
+        end
+      end
+
+      def isodoc_create
+        isodoc = Util::load_isodoc(@doctype)
+        isodoc.i18n_init(@lang, @script, @locale) # read in internationalisation
+        isodoc.metadata_init(@lang, @script, @locale, isodoc.i18n)
+        isodoc.info(@xml, nil)
+        isodoc
+      end
+
+      # create the @meta class of isodoc, for populating Liquid,
+      # with "navigation" set to the index bar.
+      # extracted from the manifest
+      def isodoc_populate
+        @isodoc.info(@xml, nil)
+        { navigation: indexfile(@manifest), nav_object: index_object(@manifest),
+          docrefs: liquid_docrefs(@manifest),
+          "prefatory-content": isodoc_builder(@xml.at("//prefatory-content")),
+          "final-content": isodoc_builder(@xml.at("//final-content")),
+          doctitle: @bibdata.title.first.title.content,
+          docnumber: @bibdata.docidentifier.first.id }.each do |k, v|
+          v and @isodoc.meta.set(k, v)
+        end
+      end
+
+      def isodoc_builder(node)
+        node or return
+
+        # Kludging namespace back in because of Shale brain damage
+        doc = Nokogiri::XML(node.to_xml.sub(">", " xmlns='http://www.metanorma.org'>"))
+        Nokogiri::HTML::Builder.new(encoding: "UTF-8") do |b|
+          b.div do |div|
+            doc.root.children&.each { |n| @isodoc.parse(n, div) }
+          end
+        end.doc.root.to_html
+      end
+
+      def ns(xpath)
+        @isodoc.ns(xpath)
+      end
+    end
+  end
+end

data/lib/metanorma/collection/sectionsplit/sectionsplit.rb

@@ -0,0 +1,218 @@
+require "yaml"
+require_relative "../../util/util"
+require_relative "../xrefprocess/xrefprocess"
+
+module Metanorma
+  class Collection
+    class Sectionsplit
+      attr_accessor :filecache, :key
+
+      def initialize(opts)
+        @input_filename = opts[:input]
+        @base = opts[:base]
+        @output_filename = opts[:output]
+        @xml = opts[:xml]
+        @dir = opts[:dir]
+        @compile_opts = opts[:compile_opts] || {}
+        @fileslookup = opts[:fileslookup]
+        @ident = opts[:ident]
+        @isodoc = opts[:isodoc]
+      end
+
+      def ns(xpath)
+        @isodoc.ns(xpath)
+      end
+
+      def build_collection
+        collection_setup(@base, @dir)
+        files = sectionsplit # (@input_filename, @base, @dir, @compile_opts)
+        input_xml = Nokogiri::XML(File.read(@input_filename,
+                                            encoding: "UTF-8"), &:huge)
+        collection_manifest(@base, files, input_xml, @xml, @dir).render(
+          { format: %i(html), output_folder: "#{@output_filename}_collection",
+            coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
+        )
+      end
+
+      def collection_manifest(filename, files, origxml, _presxml, dir)
+        File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
+          f.write(collectionyaml(files, origxml))
+        end
+        Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
+      end
+
+      def collection_setup(filename, dir)
+        FileUtils.mkdir_p "#{filename}_collection" if filename
+        FileUtils.mkdir_p dir
+        File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
+          f.write(coll_cover)
+        end
+      end
+
+      def coll_cover
+        <<~COVER
+          <html><head><meta charset="UTF-8"/></head><body>
+                <h1>{{ doctitle }}</h1>
+                <h2>{{ docnumber }}</h2>
+                <nav>{{ navigation }}</nav>
+              </body></html>
+        COVER
+      end
+
+      SPLITSECTIONS =
+        [["//preface/*", "preface"], ["//sections/*", "sections"],
+         ["//annex", nil],
+         ["//bibliography/*[not(@hidden = 'true')]", "bibliography"],
+         ["//indexsect", nil], ["//colophon", nil]].freeze
+
+      # Input XML is Semantic
+      # def sectionsplit(filename, basename, dir, compile_options, fileslookup = nil, ident = nil)
+      def sectionsplit
+        xml = sectionsplit_prep(File.read(@input_filename), @base, @dir)
+        @key = Metanorma::Collection::XrefProcess::xref_preprocess(xml, @isodoc)
+        SPLITSECTIONS.each_with_object([]) do |n, ret|
+          conflate_floatingtitles(xml.xpath(ns(n[0]))).each do |s|
+            ret << sectionfile(xml, emptydoc(xml), "#{@base}.#{ret.size}", s,
+                               n[1])
+          end
+        end
+      end
+
+      def block?(node)
+        %w(p table formula admonition ol ul dl figure quote sourcecode example
+           pre note pagebrreak hr bookmark requirement recommendation permission
+           svgmap inputform toc passthrough review imagemap).include?(node.name)
+      end
+
+      def conflate_floatingtitles(nodes)
+        holdover = false
+        nodes.each_with_object([]) do |x, m|
+          if holdover then m.last << x
+          else m << [x]
+          end
+          holdover = block?(x)
+        end
+      end
+
+      def sectionsplit_prep(file, filename, dir)
+        @splitdir = dir
+        xml1filename, type = sectionsplit_preprocess_semxml(file, filename)
+        Compile.new.compile(
+          xml1filename,
+          { format: :asciidoc, extension_keys: [:presentation], type: type }
+         .merge(@compile_opts),
+        )
+        Nokogiri::XML(File.read(xml1filename.sub(/\.xml$/, ".presentation.xml"),
+                                encoding: "utf-8"), &:huge)
+      end
+
+      def sectionsplit_preprocess_semxml(file, filename)
+        xml = Nokogiri::XML(file, &:huge)
+        type = xml.root.name.sub("-standard", "").to_sym
+        sectionsplit_update_xrefs(xml)
+        xml1 = sectionsplit_write_semxml(filename, xml)
+        @filecache ||= []
+        @filecache << xml1
+        [xml1.path, type]
+      end
+
+      def sectionsplit_update_xrefs(xml)
+        if c = @fileslookup&.parent
+          n = c.nested
+          c.nested = true # so unresolved erefs are not deleted
+          c.update_xrefs(xml, @ident, {})
+          c.nested = n
+        end
+      end
+
+      def sectionsplit_write_semxml(filename, xml)
+        Tempfile.open([filename, ".xml"], encoding: "utf-8") do |f|
+          # f.write(@isodoc.to_xml(svg_preprocess(xml)))
+          f.write(@isodoc.to_xml(xml))
+          f
+        end
+      end
+
+      def emptydoc(xml)
+        out = xml.dup
+        out.xpath(
+          ns("//preface | //sections | //annex | //bibliography/clause | " \
+             "//bibliography/references[not(@hidden = 'true')] | //indexsect | " \
+             "//colophon"),
+        ).each(&:remove)
+        out
+      end
+
+      def sectionfile(fulldoc, xml, file, chunks, parentnode)
+        fname = create_sectionfile(fulldoc, xml.dup, file, chunks, parentnode)
+        { order: chunks.last["displayorder"].to_i, url: fname,
+          title: titlerender(chunks.last) }
+      end
+
+      def create_sectionfile(xml, out, file, chunks, parentnode)
+        ins = out.at(ns("//metanorma-extension")) || out.at(ns("//bibdata"))
+        sectionfile_insert(ins, chunks, parentnode)
+        Metanorma::Collection::XrefProcess::xref_process(out, xml, @key, @ident, @isodoc)
+        outname = "#{file}.xml"
+        File.open(File.join(@splitdir, outname), "w:UTF-8") do |f|
+          f.write(out)
+        end
+        outname
+      end
+
+      def sectionfile_insert(ins, chunks, parentnode)
+        if parentnode
+          ins.next = "<#{parentnode}/>"
+          chunks.each { |c| ins.next.add_child(c.dup) }
+        else chunks.each { |c| ins.next = c.dup }
+        end
+      end
+
+      def titlerender(section)
+        title = section.at(ns("./title")) or return "[Untitled]"
+        t = title.dup
+        t.xpath(ns(".//tab | .//br")).each { |x| x.replace(" ") }
+        t.xpath(ns(".//strong")).each { |x| x.replace(x.children) }
+        t.children.to_xml
+      end
+
+      def collectionyaml(files, xml)
+        ret = {
+          directives: ["presentation-xml", "bare-after-first"],
+          bibdata: {
+            title: {
+              type: "title-main", language: @lang,
+              content: xml.at(ns("//bibdata/title")).text
+            },
+            type: "collection",
+            docid: {
+              type: xml.at(ns("//bibdata/docidentifier/@type")).text,
+              id: xml.at(ns("//bibdata/docidentifier")).text,
+            },
+          },
+          manifest: {
+            level: "collection", title: "Collection",
+            docref: files.sort_by { |f| f[:order] }.each.map do |f|
+              { fileref: f[:url], identifier: f[:title] }
+            end
+          },
+        }
+        ::Metanorma::Util::recursive_string_keys(ret).to_yaml
+      end
+
+      def section_split_cover(col, ident, _one_doc_coll)
+        dir = File.dirname(col.file)
+        collection_setup(nil, dir)
+        r = ::Metanorma::Collection::Renderer
+          .new(col, dir, output_folder: "#{ident}_collection",
+                         format: %i(html), coverpage: File.join(dir, "cover.html"))
+        r.coverpage
+        # filename = one_doc_coll ? "#{ident}_index.html" : "index.html"
+        filename = File.basename("#{ident}_index.html") # ident can be a directory with YAML indirection
+        FileUtils.mv File.join(r.outdir, "index.html"), File.join(dir, filename)
+        FileUtils.rm_rf r.outdir
+        filename
+      end
+    end
+  end
+end

data/lib/metanorma/collection/util/disambig_files.rb

@@ -0,0 +1,37 @@
+module Metanorma
+  class Collection
+    module Util
+      class DisambigFiles
+        def initialize
+          @seen_filenames = []
+        end
+
+        def strip_root(name)
+          name.sub(%r{^(\./)?(\.\./)+}, "")
+        end
+
+        def source2dest_filename(name, disambig = true)
+          n = strip_root(name)
+          dir = File.dirname(n)
+          base = File.basename(n)
+          if disambig && @seen_filenames.include?(base)
+            base = disambiguate_filename(base)
+          end
+          @seen_filenames << base
+          dir == "." ? base : File.join(dir, base)
+        end
+
+        def disambiguate_filename(base)
+          m = /^(?<start>.+\.)(?!0)(?<num>\d+)\.(?<suff>[^.]*)$/.match(base) ||
+            /^(?<start>.+\.)(?<suff>[^.]*)/.match(base) ||
+            /^(?<start>.+)$/.match(base)
+          i = m.names.include?("num") ? m["num"].to_i + 1 : 1
+          while @seen_filenames.include? base = "#{m['start']}#{i}.#{m['suff']}"
+            i += 1
+          end
+          base
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/collection/util/util.rb

@@ -0,0 +1,72 @@
+module Metanorma
+  class Collection
+    module Util
+      class << self
+        def gather_bibitems(xml)
+          xml.xpath("//xmlns:bibitem[@id]").each_with_object({}) do |b, m|
+            if m[b["id"]]
+              b.remove
+              next
+              # we can't update duplicate bibitem, processing updates wrong one
+            else
+              m[b["id"]] = b
+            end
+          end
+        end
+
+        def gather_bibitemids(xml)
+          xml.xpath("//*[@bibitemid]").each_with_object({}) do |e, m|
+            /^semantic__/.match?(e.name) and next
+            m[e["bibitemid"]] ||= []
+            m[e["bibitemid"]] << e
+          end
+        end
+
+        def gather_citeases(xml)
+          xml.xpath("//*[@citeas]").each_with_object({}) do |e, m|
+            /^semantic__/.match?(e.name) and next
+            m[e["citeas"]] ||= []
+            m[e["citeas"]] << e
+          end
+        end
+
+        def add_suffix_to_attributes(doc, suffix, tag_name, attr_name,
+isodoc)
+          (suffix.nil? || suffix.empty?) and return
+          doc.xpath(isodoc.ns("//#{tag_name}[@#{attr_name}]")).each do |elem|
+            a = elem.attributes[attr_name].value
+            /_#{suffix}$/.match?(a) or
+              elem.attributes[attr_name].value = "#{a}_#{suffix}"
+          end
+        end
+
+        def hash_key_detect(directives, key, variable)
+          c = directives.detect { |x| x.key == key } or
+            return variable
+          c.value
+        end
+
+        def rel_path_resolve(dir, path)
+          path.nil? and return path
+          path.empty? and return path
+          p = Pathname.new(path)
+          p.absolute? ? path : File.join(dir, path)
+        end
+
+        def key(ident)
+          @c ||= HTMLEntities.new
+          @c.decode(ident).gsub(/(\p{Zs})+/, " ")
+        end
+
+        class Dummy
+          def attr(_key); end
+        end
+
+        def load_isodoc(doctype)
+          x = Asciidoctor.load nil, backend: doctype.to_sym
+          x.converter.html_converter(Dummy.new) # to obtain Isodoc class
+        end
+      end
+    end
+  end
+end