metanorma 1.7.7 → 2.0.0

data/lib/metanorma/collection_fileparse.rb

@@ -1,257 +0,0 @@
-module Metanorma
-  # XML collection renderer
-  class CollectionRenderer
-    # Resolves references to other files in the collection. Three routines:
-    # 1. Eref to a document that has been split into multiple documents
-    # (sectionsplit) are resolved to direct eref to the split document
-    # 2. Indirect erefs to a file anchor in an unknwon file in the collection
-    # (bibitem[@type = 'internal'] ) are resolved to direct eref to the
-    # containing document
-    # 3. Direct erefs to other files in collection
-    # (repo(current-metanorma-collection/x) are resolved to hyperlinks
-    # @param file [String] XML content
-    # @param identifier [String] docid
-    # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to
-    #   anchor to filename
-    # @return [String] XML content
-    def update_xrefs(file, docid, internal_refs)
-      xml, sso = update_xrefs_prep(file, docid)
-      @nested || sso or Metanorma::XrefProcess::xref_process(xml, xml, nil, docid, @isodoc)
-      @nested or update_indirect_refs_to_docs(xml, docid, internal_refs)
-      @files.add_document_suffix(docid, xml)
-      @nested or update_sectionsplit_refs_to_docs(xml, internal_refs)
-      update_direct_refs_to_docs(xml, docid)
-      hide_refs(xml)
-      sso and eref2link(xml)
-      svgmap_resolve(datauri_encode(xml), @files.get(docid, :ids))
-      xml.to_xml
-    end
-
-    def update_xrefs_prep(file, docid)
-      docxml = file.is_a?(String) ? Nokogiri::XML(file, &:huge) : file
-      supply_repo_ids(docxml)
-      sso = @files.get(docid, :sectionsplit_output)
-      [docxml, sso]
-    end
-
-    def update_sectionsplit_refs_to_docs(docxml, internal_refs)
-      Util::gather_citeases(docxml).each do |k, v|
-        (@files.get(k) && @files.get(k, :sectionsplit)) or next
-        opts = { key: @files.get(k, :indirect_key),
-                 source_suffix: docxml.root["document_suffix"],
-                 target_suffix: @files.get(k, :document_suffix) }
-        refs = v.each_with_object({}) do |eref, m|
-          update_sectionsplit_eref_to_doc(eref, internal_refs, m, opts)
-        end
-        add_hidden_bibliography(docxml, refs)
-      end
-    end
-
-    def update_sectionsplit_eref_to_doc(eref, internal_refs, doclist, opts)
-      a = eref.at(ns("./localityStack/locality[@type = 'anchor']/" \
-                     "referenceFrom")) or return
-      doc = internal_refs[opts[:key]]["#{a.text}_#{opts[:target_suffix]}"]
-      bibitemid = Metanorma::Utils::to_ncname("#{doc}_#{opts[:source_suffix]}")
-      eref["bibitemid"] = bibitemid
-      doclist[bibitemid] ||= doc
-      doclist
-    end
-
-    def new_hidden_ref(xmldoc)
-      ins = xmldoc.at(ns("bibliography")) or
-        xmldoc.root << "<bibliography/>" and ins = xmldoc.at(ns("bibliography"))
-      ins.at(ns("./referenced[@hidden = 'true']")) or
-        ins.add_child("<references hidden='true' normative='false'/>").first
-    end
-
-    def add_hidden_bibliography(xmldoc, refs)
-      ins = new_hidden_ref(xmldoc)
-      refs.each do |k, v|
-        url = @files.url(v, {})
-        ins << <<~XML
-          <bibitem id="#{k}">
-            <docidentifier type="repository">current-metanorma-collection/#{v}</docidentifier>
-            <uri type='citation'>#{url}</uri>
-          </bibitem>
-        XML
-      end
-    end
-
-    def eref2link(docxml)
-      isodoc = IsoDoc::PresentationXMLConvert.new({})
-      isodoc.bibitem_lookup(docxml)
-      isodoc.eref2link(docxml)
-    end
-
-    BIBITEM_NOT_REPO_XPATH = "//bibitem[not(ancestor::bibitem)]" \
-      "[not(./docidentifier[@type = 'repository'])]".freeze
-
-    def supply_repo_ids(doc)
-      doc.xpath(ns(BIBITEM_NOT_REPO_XPATH)).each do |b|
-        b.xpath(ns("./docidentifier")).each do |docid|
-          id = @isodoc.docid_prefix(docid["type"], docid.children.to_xml)
-          @files.get(id) or next
-          @files.get(id, :indirect_key) and next # will resolve as indirect key
-          docid.next = "<docidentifier type='repository'>" \
-                       "current-metanorma-collection/#{id}</docidentifier>"
-        end
-      end
-    end
-
-    def svgmap_resolve(docxml, ids)
-      isodoc = IsoDoc::PresentationXMLConvert.new({})
-      isodoc.bibitem_lookup(docxml)
-      docxml.xpath(ns("//svgmap//eref")).each do |e|
-        svgmap_resolve1(e, isodoc, docxml, ids)
-      end
-      Vectory::SvgMapping.new(docxml, "").call
-      docxml.xpath(ns("//svgmap")).each { |s| isodoc.svgmap_extract(s) }
-    end
-
-    def svgmap_resolve1(eref, isodoc, _docxml, ids)
-      href = isodoc.eref_target(eref) or return
-      href == "##{eref['bibitemid']}" ||
-        (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
-      eref["target"] = href.strip
-      eref.name = "link"
-      eref.elements&.remove
-    end
-
-    # repo(current-metanorma-collection/ISO 17301-1:2016)
-    # replaced by bibdata of "ISO 17301-1:2016" in situ as bibitem.
-    # Any erefs to that bibitem id are replaced with relative URL
-    # Preferably with anchor, and is a job to realise dynamic lookup
-    # of localities.
-    def update_direct_refs_to_docs(docxml, identifier)
-      erefs, erefs1 = update_direct_refs_to_docs_prep(docxml)
-      docxml.xpath(ns("//bibitem")).each do |b|
-        docid = b.at(ns("./docidentifier[@type = 'repository']")) or next
-        strip_unresolved_repo_erefs(identifier, docid, erefs1, b) or next
-        update_bibitem(b, identifier)
-        docid = docid_to_citeas(b) or next
-        erefs[docid] and update_anchors(b, docid, erefs[docid])
-      end
-    end
-
-    def update_direct_refs_to_docs_prep(docxml)
-      @ncnames = {}
-      [Util::gather_citeases(docxml), Util::gather_bibitemids(docxml)]
-    end
-
-    # strip erefs if they are repository erefs, but do not point to a document
-    # within the current collection. This can happen if a collection consists
-    # of many documents, but not all are included in the current collection.
-    # Do not do this if this is a sectionsplit collection or a nested manifest.
-    # Return false if bibitem is not to be further processed
-    def strip_unresolved_repo_erefs(_document_id, bib_docid, erefs, bibitem)
-      %r{^current-metanorma-collection/(?!Missing:)}.match?(bib_docid.text) and
-        return true
-      @nested and return false
-      erefs[bibitem["id"]]&.each { |x| x.parent and strip_eref(x) }
-      false
-    end
-
-    # Resolve erefs to a container of ids in another doc,
-    # to an anchor eref (direct link)
-    def update_indirect_refs_to_docs(docxml, _docidentifier, internal_refs)
-      bibitems, erefs = update_indirect_refs_to_docs_prep(docxml)
-      internal_refs.each do |schema, ids|
-        ids.each do |id, file|
-          k = indirect_ref_key(schema, id, docxml)
-          update_indirect_refs_to_docs1(docxml, k, file, bibitems, erefs)
-        end
-      end
-    end
-
-    def update_indirect_refs_to_docs_prep(docxml)
-      bibitems = Util::gather_bibitems(docxml)
-      erefs = Util::gather_bibitemids(docxml)
-      @updated_anchors = {}
-      [bibitems, erefs]
-    end
-
-    def indirect_ref_key(schema, id, docxml)
-      /^#{schema}_/.match?(id) and return id
-      ret = "#{schema}_#{id}"
-      suffix = docxml.root["document_suffix"]
-      (k = docxml.root["type"]) && k != schema && suffix and
-        ret = "#{ret}_#{suffix}"
-      ret
-    end
-
-    def update_indirect_refs_to_docs1(_docxml, key, file, bibitems, erefs)
-      erefs[key]&.each do |e|
-        e["citeas"] = file
-        update_indirect_refs_to_docs_anchor(e, file)
-      end
-      update_indirect_refs_to_docs_docid(bibitems[key], file)
-    end
-
-    def update_indirect_refs_to_docs_anchor(eref, file)
-      a = eref.at(ns(".//locality[@type = 'anchor']/referenceFrom")) or return
-      suffix = file
-      @files.get(file) && p = @files.get(file, :parentid) and
-        suffix = "#{p}_#{suffix}"
-      existing = a.text
-      anchor = existing
-      @files.url?(file) or
-        anchor = Metanorma::Utils::to_ncname("#{anchor}_#{suffix}")
-      @updated_anchors[existing] or a.children = anchor
-      @updated_anchors[anchor] = true
-    end
-
-    def update_indirect_refs_to_docs_docid(bibitem, file)
-      docid = bibitem&.at(ns("./docidentifier[@type = 'repository']")) or
-        return
-      docid.children = "current-metanorma-collection/#{file}"
-      docid.previous =
-        "<docidentifier type='metanorma-collection'>#{file}</docidentifier>"
-    end
-
-    # update crossrefences to other documents, to include
-    # disambiguating document suffix on id
-    def update_anchors(bib, docid, erefs)
-      erefs.each do |e|
-        if @files.get(docid) then update_anchor_loc(bib, e, docid)
-        else
-          msg = "<strong>** Unresolved reference to document #{docid} " \
-                "from eref</strong>"
-          e << msg
-          strip_eref(e)
-          @log&.add("Cross-References", e, msg)
-        end
-      end
-    end
-
-    def update_anchor_loc(bib, eref, docid)
-      loc = eref.at(".//xmlns:locality[@type = 'anchor']") or
-        return update_anchor_create_loc(bib, eref, docid)
-      ref = loc.at("./xmlns:referenceFrom") or return
-      anchor = suffix_anchor(ref, docid)
-      a = @files.get(docid, :anchors) or return
-      a.inject([]) do |m, (_, x)|
-        m += x.values
-      end.include?(anchor) or return
-      ref.content = anchor
-    end
-
-    def suffix_anchor(ref, docid)
-      @ncnames[docid] ||= Metanorma::Utils::to_ncname(docid)
-      anchor = ref.text
-      @files.url?(docid) or anchor = "#{@ncnames[docid]}_#{anchor}"
-      anchor
-    end
-
-    # if there is a crossref to another document, with no anchor, retrieve the
-    # anchor given the locality, and insert it into the crossref
-    def update_anchor_create_loc(_bib, eref, docid)
-      ins = eref.at(ns("./localityStack")) or return
-      type = ins.at(ns("./locality/@type"))&.text
-      type = "clause" if type == "annex"
-      ref = ins.at(ns("./locality/referenceFrom"))&.text
-      anchor = @files.get(docid, :anchors).dig(type, ref) or return
-      ins << "<locality type='anchor'><referenceFrom>#{anchor.sub(/^_/, '')}" \
-             "</referenceFrom></locality>"
-    end
-  end
-end

data/lib/metanorma/collection_fileprocess.rb

@@ -1,168 +0,0 @@
-# frozen_string_literal: true
-
-require "isodoc"
-require "metanorma-utils"
-require_relative "collection_fileparse"
-
-module Metanorma
-  # XML collection renderer
-  class CollectionRenderer
-    # compile and output individual file in collection
-    # warn "metanorma compile -x html #{f.path}"
-    def file_compile(file, filename, identifier)
-      return if @files.get(identifier, :sectionsplit) == "true"
-
-      opts = {
-        format: :asciidoc,
-        extension_keys: @format,
-        output_dir: @outdir,
-      }.merge(compile_options_update(identifier))
-
-      @compile.compile file, opts
-      @files.set(identifier, :outputs, {})
-      file_compile_formats(filename, identifier)
-    end
-
-    def compile_options_update(identifier)
-      ret = @compile_options.dup
-      Array(@directives).include?("presentation-xml") ||
-        @files.get(identifier, :presentationxml) and
-        ret.merge!(passthrough_presentation_xml: true)
-      @files.get(identifier, :sectionsplit) == "true" and
-        ret.merge!(sectionsplit: "true")
-      @files.get(identifier, :bare) == true and
-        ret.merge!(bare: true)
-      ret
-    end
-
-    def file_compile_formats(filename, identifier)
-      f = @files.get(identifier, :outputs)
-      @format << :presentation if @format.include?(:pdf)
-      @format.each do |e|
-        ext = @compile.processor.output_formats[e]
-        fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
-        (/html$/.match?(ext) && @files.get(identifier, :sectionsplit)) or
-          f[e] = File.join(@outdir, fn)
-      end
-      @files.set(identifier, :outputs, f)
-    end
-
-    def copy_file_to_dest(identifier)
-      dest = File.join(@outdir, @files.get(identifier, :out_path))
-      FileUtils.mkdir_p(File.dirname(dest))
-      FileUtils.cp @files.get(identifier, :ref), dest
-    end
-
-    # process each file in the collection
-    # files are held in memory, and altered as postprocessing
-    def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"
-      internal_refs = locate_internal_refs
-      @files.keys.each_with_index do |ident, i|
-        i.positive? && Array(@directives).include?("bare-after-first") and
-          @compile_options.merge!(bare: true)
-        if @files.get(ident, :attachment) then copy_file_to_dest(ident)
-        else
-          file, filename = @files.targetfile_id(ident, read: true)
-          warn "\n\n\n\n\nProcess #{filename}: #{DateTime.now.strftime('%H:%M:%S')}"
-          collection_xml = update_xrefs(file, ident, internal_refs)
-          collection_filename = File.basename(filename, File.extname(filename))
-          collection_xml_path = File.join(Dir.tmpdir,
-                                          "#{collection_filename}.xml")
-          File.write collection_xml_path, collection_xml, encoding: "UTF-8"
-          file_compile(collection_xml_path, filename, ident)
-          FileUtils.rm(collection_xml_path)
-        end
-      end
-    end
-
-    # gather internal bibitem references
-    def gather_internal_refs
-      @files.keys.each_with_object({}) do |i, refs|
-        @files.get(i, :attachment) and next
-        file, = @files.targetfile_id(i, read: true)
-        gather_internal_refs1(file, i, refs)
-      end
-    end
-
-    def gather_internal_refs1(file, ident, refs)
-      f = Nokogiri::XML(file, &:huge)
-      !@files.get(ident, :sectionsplit) and
-        gather_internal_refs_indirect(f, refs)
-      key = @files.get(ident, :indirect_key) and
-        gather_internal_refs_sectionsplit(f, ident, key, refs)
-    end
-
-    def gather_internal_refs_indirect(doc, refs)
-      doc.xpath(ns("//bibitem[@type = 'internal']/" \
-                   "docidentifier[@type = 'repository']")).each do |d|
-                     a = d.text.split(%r{/}, 2)
-                     a.size > 1 or next
-                     refs[a[0]] ||= {}
-                     refs[a[0]][a[1]] = false
-                   end
-    end
-
-    def gather_internal_refs_sectionsplit(_doc, ident, key, refs)
-      refs[key] ||= {}
-      @files.get(ident, :ids).each_key do |k|
-        refs[key][k] = false
-      end
-    end
-
-    def populate_internal_refs(refs)
-      @files.keys.reject do |k|
-        @files.get(k, :attachment) || @files.get(k, :sectionsplit)
-      end.each do |ident|
-        locate_internal_refs1(refs, ident, @isodoc.docid_prefix("", ident.dup))
-      end
-      refs
-    end
-
-    # resolve file location for the target of each internal reference
-    def locate_internal_refs
-      warn "\n\n\n\n\nInternal Refs: #{DateTime.now.strftime('%H:%M:%S')}"
-      refs = populate_internal_refs(gather_internal_refs)
-      refs.each do |schema, ids|
-        ids.each do |id, key|
-          key and next
-          refs[schema][id] = "Missing:#{schema}:#{id}"
-          @log&.add("Cross-References", nil, refs[schema][id])
-        end
-      end
-      refs
-    end
-
-    def locate_internal_refs1(refs, identifier, ident)
-      file, = @files.targetfile_id(ident, read: true)
-      t = locate_internal_refs1_prep(file)
-      refs.each do |schema, ids|
-        ids.keys.select { |id| t[id] }.each do |id|
-          t[id].at("./ancestor-or-self::*[@type = '#{schema}']") and
-            refs[schema][id] = identifier
-        end
-      end
-    end
-
-    def locate_internal_refs1_prep(file)
-      xml = Nokogiri::XML(file, &:huge)
-      r = xml.root["document_suffix"]
-      xml.xpath("//*[@id]").each_with_object({}) do |i, x|
-        /^semantic_/.match?(i.name) and next
-        x[i["id"]] = i
-        r and x[i["id"].sub(/_#{r}$/, "")] = i
-      end
-    end
-
-    def update_bibitem(bib, identifier)
-      docid = get_bibitem_docid(bib, identifier) or return
-      newbib = dup_bibitem(docid, bib)
-      url = @files.url(docid, relative: true,
-                              doc: !@files.get(docid, :attachment))
-      dest = newbib.at("./docidentifier") || newbib.at(ns("./docidentifier"))
-      dest or dest = newbib.elements[-1]
-      dest.previous = "<uri type='citation'>#{url}</uri>"
-      bib.replace(newbib)
-    end
-  end
-end

data/lib/metanorma/collection_manifest.rb

@@ -1,168 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "util"
-
-module Metanorma
-  # Metanorma collection's manifest
-  class CollectionManifest
-    # @return [Metanorma::Collection]
-    attr_reader :collection
-
-    # @param level [String]
-    # @param title [String, nil]
-    # @param docref [Array<Hash{String=>String}>]
-    # @param manifest [Array<Metanorma::CollectionManifest>]
-    def initialize(level, title = nil, docref = [], manifest = [])
-      @level = level
-      @title = title
-      @docref = docref
-      @manifest = manifest
-      @disambig = Util::DisambigFiles.new
-    end
-
-    class << self
-      # @param mnf [Nokogiri::XML::Element]
-      # @return [Metanorma::CollectionManifest]
-      def from_yaml(mnf)
-        manifest = RelatonBib.array(mnf["manifest"]).map do |m|
-          from_yaml m
-        end
-        docref = RelatonBib.array mnf["docref"]
-        new(mnf["level"], mnf["title"], parse_docrefs_yaml(docref), manifest)
-      end
-
-      # @param mnf [Nokogiri::XML::Element]
-      # @return [Metanorma::CollectionManifest]
-      def from_xml(mnf)
-        level = mnf.at("level").text
-        title = mnf.at("title")&.text
-        manifest = mnf.xpath("xmlns:manifest").map { |m| from_xml(m) }
-        new(level, title, parse_docrefs_xml(mnf), manifest)
-      end
-
-      private
-
-      def parse_docrefs_yaml(docrefs)
-        docrefs.map do |dr|
-          h = {}
-          h["identifier"] = dr["identifier"] ||
-            UUIDTools::UUID.random_create.to_s
-          dr["manifest"] and h["manifest"] = from_yaml(dr["manifest"].first)
-          %w(fileref url attachment sectionsplit index presentation-xml).each do |k|
-            dr.has_key?(k) and h[k] = dr[k]
-          end
-          h
-        end
-      end
-
-      # @param mnf [Nokogiri::XML::Element]
-      # @return [Hash{String=>String}]
-      def parse_docrefs_xml(mnf)
-        mnf.xpath("xmlns:docref").map do |dr|
-          h = { "identifier" => parse_docrefs_xml_id(dr) }
-          %i(fileref url attachment sectionsplit index).each do |s|
-            h[s.to_s] = dr[s] if dr[s]
-          end
-          m = dr.at("manifest") and h["manifest"] = from_xml(m)
-          h["presentation-xml"] = dr[:presentationxml] if dr[:presentationxml]
-          h
-        end
-      end
-
-      def parse_docrefs_xml_id(docref)
-        if i = docref.at("identifier")
-          i.children.to_xml
-        else UUIDTools::UUID.random_create
-        end
-      end
-    end
-
-    # @param col [Metanorma::Collection]
-    def collection=(col)
-      @collection = col
-      @manifest.each { |mnf| mnf.collection = col }
-    end
-
-    # @param dir [String] path to collection
-    # @return [Hash<String, Metanorma::Document>]
-    def documents(dir = "")
-      docs = @docref.each_with_object({}) do |dr, m|
-        if dr["fileref"]
-          m[Util::key dr["identifier"]] = documents_add(dir, dr)
-        elsif dr["manifest"]
-          m.merge! dr["manifest"].documents(dir)
-        end
-        m
-      end
-      @manifest.reduce(docs) { |mem, mnf| mem.merge mnf.documents(dir) }
-    end
-
-    def documents_add(dir, docref)
-      Document.parse_file(
-        Util::rel_path_resolve(dir, docref["fileref"]),
-        docref["attachment"], docref["identifier"], docref["index"]
-      )
-    end
-
-    # @param builder [Nokogiri::XML::Builder]
-    def to_xml(builder)
-      builder.manifest do |b|
-        b.level @level
-        b.title @title if @title
-        docref_to_xml b
-        @manifest&.each { |m| m.to_xml b }
-      end
-    end
-
-    # @return [Array<Hash{String=>String}>]
-    def docrefs
-      return @docrefs if @docrefs
-
-      drfs = @docref.map { |dr| dr }
-      @manifest.reduce(drfs) { |mem, mnf| mem + mnf.docrefs }
-    end
-
-    def docref_by_id(docid)
-      refs = docrefs
-      dref = refs.detect { |k| k["identifier"] == docid }
-      dref || docrefs.detect { |k| /^#{k['identifier']}/ =~ docid }
-    end
-
-    private
-
-    # @param builder [Nokogiri::XML::Builder]
-    def docref_to_xml(builder)
-      @disambig = Util::DisambigFiles.new
-      @docref.each do |dr|
-        drf = builder.docref do |b|
-          b.identifier { |i| i << dr["identifier"] }
-          !dr["attachment"] && !dr["sectionsplit"] && @collection &&
-            d = @collection.bibdatas[Util::key dr["identifier"]] and
-            b.parent.add_child(d.bibitem.to_xml(bibdata: true))
-          m = dr["manifest"] and m.to_xml b
-        end
-        docref_to_xml_attrs(drf, dr)
-      end
-    end
-
-    def docref_to_xml_attrs(elem, docref)
-      f = docref["fileref"] and elem[:fileref] = @disambig.strip_root(f)
-      %i(attachment sectionsplit url).each do |i|
-        elem[i] = docref[i.to_s] if docref[i.to_s]
-      end
-      elem[:index] = docref.has_key?("index") ? docref["index"] : "true"
-      elem[:presentationxml] = "true" if docref["presentation-xml"] &&
-        [true, "true"].include?(docref["presentation-xml"])
-      docref_to_xml_attrs_id(elem, docref)
-    end
-
-    def docref_to_xml_attrs_id(elem, docref)
-      if collection&.directives&.include?("documents-inline")
-        id = collection.documents.find_index do |k, _|
-          k == docref["identifier"]
-        end
-        id and elem[:id] = format("doc%<index>09d", index: id)
-      end
-    end
-  end
-end