metanorma 1.7.7 → 2.0.0

data/lib/metanorma/collection_xref_process.rb

@@ -1,217 +0,0 @@
-module Metanorma
-  module XrefProcess
-    class << self
-      def xref_preprocess(xml, isodoc)
-        @isodoc = isodoc
-        key = (0...8).map { rand(65..90).chr }.join # random string
-        xml.root["type"] = key # to force recognition of internal refs
-        # bookmarks etc as new id elements introduced in Presentation XML:
-        # add doc suffix
-        Metanorma::Utils::anchor_attributes.each do |(tag_name, attribute_name)|
-          Util::add_suffix_to_attributes(xml, xml.root["document_suffix"],
-                                         tag_name, attribute_name, isodoc)
-        end
-        key
-      end
-
-      def ns(xpath)
-        @isodoc.ns(xpath)
-      end
-
-      def xref_process(section, xml, key, ident, isodoc)
-        @isodoc ||= isodoc
-        svg_preprocess(section, Metanorma::Utils::to_ncname(ident))
-        refs = eref_to_internal_eref(section, xml, key)
-        refs += xref_to_internal_eref(section, xml, key)
-        ins = new_hidden_ref(section)
-        copied_refs = copy_repo_items_biblio(ins, section, xml)
-        insert_indirect_biblio(ins, refs - copied_refs, key, xml)
-      end
-
-      def svg_preprocess(xml, doc_suffix)
-        suffix = doc_suffix.nil? || doc_suffix.blank? ? "" : "_#{doc_suffix}"
-        xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
-          m = svgmap_wrap(s)
-          svg_xrefs(s, m, suffix)
-        end
-        xml
-      end
-
-      def svgmap_wrap(svg)
-        ret = svg.at("./ancestor::xmlns:svgmap") and return ret
-        ret = svg.at("./ancestor::xmlns:figure")
-        ret.wrap("<svgmap/>")
-        svg.at("./ancestor::xmlns:svgmap")
-      end
-
-      def svg_xrefs(svg, svgmap, suffix)
-        svg.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
-          /^#/.match? a["href"] or next
-          a["href"] = a["href"].sub(/^#/, "")
-          svgmap << "<target href='#{a['href']}'>" \
-            "<xref target='#{a['href']}#{suffix}'/></target>"
-        end
-      end
-
-      def make_anchor(elem, anchor)
-        elem.at(ns("./localityStack | ./locality")) and return
-        elem.text.strip.empty? and elem << anchor
-        elem <<
-          "<localityStack><locality type='anchor'><referenceFrom>" \
-            "#{anchor}</referenceFrom></locality></localityStack>"
-      end
-
-      def xref_to_internal_eref(section, xml, key)
-        key or return [] # no sectionsplit, no playing with xrefs
-        bibitems, indirect = xref_to_internal_eref_prep(section, xml)
-        section.xpath(ns("//xref")).each_with_object({}) do |x, m|
-          xref_prefix_key(x, key, indirect)
-          x["bibitemid"] = x["target"]
-          m[x["bibitemid"]] = true
-          xref_to_internal_eref_anchor(x, key, bibitems,
-                                       xml.root["document_suffix"])
-        end.keys
-      end
-
-      def xref_to_internal_eref_prep(section, xml)
-        bibitems = Util::gather_bibitems(section)
-        indirect_bibitems = Util::gather_bibitems(xml)
-          .select { |_, v| indirect_bib?(v) }
-        [bibitems, indirect_bibitems]
-      end
-
-      def xref_to_internal_eref_anchor(xref, key, bibitems, document_suffix)
-        t = xref["target"]
-        if d = bibitems[t]&.at(ns("./docidentifier[@type = 'repository']"))
-          m = %r{^([^/]+)}.match(d.text) and
-            t.sub!(%r(#{m[0]}_), "")
-        end
-        key and t.sub!(%r{^#{key}_}, "")
-        make_anchor(xref, t.sub(%r(_#{document_suffix}$), ""))
-        xref.delete("target")
-        xref.name = "eref"
-      end
-
-      def xref_prefix_key(xref, key, indirect)
-        if b = indirect[xref["target"]]
-          t = b.at(ns("./docidentifier[@type = 'repository']"))
-          xref["type"] = t.text.sub(%r{/.*$}, "")
-        elsif key
-          xref["target"] = "#{key}_#{xref['target']}"
-          xref["type"] = key
-        end
-      end
-
-      def eref_to_internal_eref(section, xml, key)
-        bibitems, indirect, bibids = eref_to_internal_eref_prep(section, xml)
-        eref_to_internal_eref_select(section, xml, bibitems)
-          .each_with_object([]) do |x, m|
-            url = bibitems[x]&.at(ns("./uri[@type = 'citation']"))&.text
-            bibids[x]&.each do |e|
-              e.at(ns("./localityStack | ./locality")) and next
-              id = eref_to_internal_eref1(e, key, url, indirect) and m << id
-            end
-          end
-      end
-
-      def eref_to_internal_eref_prep(section, xml)
-        bibitems = Util::gather_bibitems(xml)
-          .delete_if { |_, v| internal_bib?(v) }
-        indirect = Util::gather_bibitems(xml)
-          .select { |_, v| indirect_bib?(v) }
-        bibitemids = Util::gather_bibitemids(section)
-        [bibitems, indirect, bibitemids]
-      end
-
-      def eref_to_internal_eref1(elem, key, url, indirect)
-        if url
-          elem.name = "link"
-          elem["target"] = url
-          nil
-        elsif !indirect[elem["bibitemid"]]
-          nil
-        else
-          eref_to_internal_eref1_internal(elem, key, indirect)
-        end
-      end
-
-      def eref_to_internal_eref1_internal(elem, key, indirect)
-        t = elem["bibitemid"]
-        if key
-          t = "#{key}_#{t}"
-          elem["type"] = key
-        elsif d = indirect[t]&.at(ns("./docidentifier[@type = 'repository']"))
-          m = %r{^([^/]+)}.match(d.text) and
-            t.sub!(%r(#{m[0]}_), "")
-        end
-        make_anchor(elem, t)
-        elem["bibitemid"]
-      end
-
-      def eref_to_internal_eref_select(section, _xml, bibitems)
-        refs = Util::gather_bibitemids(section).keys
-        refs.uniq.reject do |x|
-          b = bibitems[x] and (indirect_bib?(b) || internal_bib?(b))
-        end
-      end
-
-      def internal_bib?(bibitem)
-        bibitem["type"] == "internal" ||
-          bibitem.at(ns("./docidentifier[@type = 'repository']"))
-      end
-
-      def indirect_bib?(bibitem)
-        a = bibitem.at(ns("./docidentifier[@type = 'repository']")) or
-          return false
-        %r{^current-metanorma-collection/}.match?(a.text) and return false
-        a.text.include?("/")
-      end
-
-      # from standoc
-      def new_hidden_ref(xmldoc)
-        ins = xmldoc.at("bibliography") or
-          xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
-        ins.add_child("<references hidden='true' normative='false'/>").first
-      end
-
-      def copy_repo_items_biblio(ins, section, xml)
-        bibitems = Util::gather_bibitems(section)
-        xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
-          .each_with_object([]) do |b, m|
-            bibitems[b["id"]] or next
-            # section.at("//*[@bibitemid = '#{b['id']}']") or next
-            ins << b.dup
-            m << b["id"]
-          end
-      end
-
-      def insert_indirect_biblio(ins, refs, key, xml)
-        refs.empty? and return
-        internal_bibitems, external_bibitems = insert_indirect_biblio_prep(xml)
-        refs.compact.reject do |x|
-          # external_bibitems[x.sub(/^#{key}_/, "")]
-        end.each do |x|
-          ins << if b = internal_bibitems[x.sub(/^#{key}_/, "")]
-                   b.dup.tap { |m| m["id"] = x }
-                 else new_indirect_bibitem(x, key)
-                 end
-        end
-      end
-
-      def insert_indirect_biblio_prep(xml)
-        bibitems = Util::gather_bibitems(xml)
-        internal_bibitems = bibitems.select { |_, v| internal_bib?(v) }
-        external_bibitems = bibitems.reject { |_, v| internal_bib?(v) }
-        [internal_bibitems, external_bibitems]
-      end
-
-      def new_indirect_bibitem(ident, prefix)
-        <<~BIBENTRY
-          <bibitem id="#{ident}" type="internal">
-          <docidentifier type="repository">#{ident.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
-          </bibitem>
-        BIBENTRY
-      end
-    end
-  end
-end

data/lib/metanorma/document.rb

@@ -1,133 +0,0 @@
-module Metanorma
-  class Document
-    # @return [Strin]
-    attr_reader :file, :attachment, :bibitem, :index
-
-    # @param bibitem [RelatonBib::BibliographicItem]
-    def initialize(bibitem, file, options = {})
-      @bibitem = bibitem
-      @file = file
-      @attachment = options[:attachment]
-      @index = options[:index]
-      @index = true if @index.nil?
-      @raw = options[:raw]
-    end
-
-    class << self
-      # @param file [String] file path
-      # @param attachment [Bool] is an attachment
-      # @param identifier [String] is the identifier assigned the file
-      # in the collection file
-      # @param index [Bool] is indication on whether to index this file in coverpage
-      # @return [Metanorma::Document]
-      def parse_file(file, attachment, identifier = nil, index = true)
-        new(bibitem(file, attachment, identifier), file,
-            { attachment: attachment, index: index })
-      end
-
-      # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
-      # @return [Metanorma::Document]
-      def parse_xml(xml)
-        new from_xml(xml)
-      end
-
-      # raw XML file, can be used to put in entire file instead of just bibitem
-      def raw_file(filename)
-        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8")) do |config|
-          config.huge
-        end
-        new(doc, filename, raw: true)
-      end
-
-      def attachment_bibitem(identifier)
-        Nokogiri::XML <<~DOCUMENT
-          <bibdata><docidentifier>#{identifier}</docidentifier></bibdata>
-        DOCUMENT
-      end
-
-      private
-
-      def mn2relaton_parser(tag)
-        case tag.sub(/-standard/, "")
-        when "bipm" then RelatonBipm::XMLParser
-        when "bsi" then RelatonBsi::XMLParser
-        when "ietf" then RelatonIetf::XMLParser
-        when "iho" then RelatonIho::XMLParser
-        when "itu" then RelatonItu::XMLParser
-        when "iec" then RelatonIec::XMLParser
-        when "iso" then RelatonIsoBib::XMLParser
-        when "nist" then RelatonNist::XMLParser
-        when "ogc" then RelatonOgc::XMLParser
-        else RelatonBib::XMLParser
-        end
-      end
-
-      # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
-      # @return [RelatonBib::BibliographicItem,RelatonIso::IsoBibliographicItem]
-      def from_xml(xml)
-        b = xml.at("//xmlns:bibitem|//xmlns:bibdata")
-        r = mn2relaton_parser(xml.root.name)
-        r.from_xml(b.to_xml)
-      end
-
-      # @param file [String]
-      # @return [Symbol] file type
-      def format(file)
-        case file
-        when /\.xml$/ then :xml
-        when /.ya?ml$/ then :yaml
-        end
-      end
-
-      # @param file [String]
-      # @return [RelatonBib::BibliographicItem,
-      #   RelatonIso::IsoBibliographicItem]
-      def bibitem(file, attachment, identifier)
-        if attachment then attachment_bibitem(identifier)
-        else
-          case format(file)
-          when :xml
-            from_xml (Nokogiri::XML(File.read(file, encoding: "UTF-8")) { |x| x.huge })
-          when :yaml
-            yaml = File.read(file, encoding: "UTF-8")
-            Relaton::Cli::YAMLConvertor.convert_single_file(yaml)
-          end
-        end
-      end
-    end
-
-    # @param builder [Nokogiri::XML::Builder, nil]
-    # @return [Nokogiri::XML::Builder, String]
-    def to_xml(builder = nil)
-      if builder
-        render_xml builder
-      else
-        Nokogiri::XML::Builder.new do |b|
-          root = render_xml b
-          root["xmlns"] = "http://metanorma.org"
-        end.to_xml
-      end
-    end
-
-    # @return [String]
-    def type
-      first = @bibitem.docidentifier.first
-      @type ||= (first&.type&.downcase ||
-                 first&.id&.match(/^[^\s]+/)&.to_s)&.downcase ||
-        "standoc"
-    end
-
-    private
-
-    def render_xml(builder)
-      if @raw
-        #builder << @bibitem.root.to_xml
-        builder.parent.add_child(@bibitem.root)
-      else
-        builder.send("#{type}-standard") do |b|
-          b << @bibitem.to_xml(bibdata: true)
-        end
-      end
-    end
-  end
-end

data/lib/metanorma/files_lookup.rb

@@ -1,224 +0,0 @@
-require "isodoc"
-require "htmlentities"
-require "metanorma-utils"
-require_relative "files_lookup_sectionsplit"
-
-module Metanorma
-  # XML collection renderer
-  class FileLookup
-    attr_accessor :files_to_delete, :parent
-
-    # hash for each document in collection of document identifier to:
-    # document reference (fileref or id), type of document reference,
-    # and bibdata entry for that file
-    # @param path [String] path to collection
-    def initialize(path, parent)
-      @c = HTMLEntities.new
-      @files = {}
-      @parent = parent
-      @xml = parent.xml
-      @isodoc = parent.isodoc
-      @path = path
-      @compile = parent.compile
-      @documents = parent.documents
-      @files_to_delete = []
-      read_files
-    end
-
-    def read_files
-      @disambig = Util::DisambigFiles.new
-      @xml.xpath(ns("//docref")).each { |d| read_file(d) }
-    end
-
-    def read_file(docref)
-      ident = docref.at(ns("./identifier"))
-      i = key(@isodoc.docid_prefix(ident["type"], ident.children.to_xml))
-      entry = file_entry(docref, ident.children.to_xml) or return
-      bibdata_process(entry, i)
-      bibitem_process(entry)
-      @files[i] = entry
-    end
-
-    def bibdata_process(entry, ident)
-      if entry[:attachment]
-        entry[:bibdata] = Metanorma::Document.attachment_bibitem(ident).root
-      else
-        file, _filename = targetfile(entry, read: true)
-        xml = Nokogiri::XML(file, &:huge)
-        add_document_suffix(ident, xml)
-        entry.merge!(anchors: read_anchors(xml), ids: read_ids(xml),
-                     bibdata: xml.at(ns("//bibdata")),
-                     document_suffix: xml.root["document_suffix"])
-      end
-    end
-
-    def bibitem_process(entry)
-      entry[:bibitem] = entry[:bibdata].dup
-      entry[:bibitem].name = "bibitem"
-      entry[:bibitem]["hidden"] = "true"
-      entry[:bibitem].at("./*[local-name() = 'ext']")&.remove
-    end
-
-    # rel_path is the source file address, determined relative to the YAML.
-    # out_path is the destination file address, with any references outside
-    # the working directory (../../...) truncated
-    # identifier is the id with only spaces, no nbsp
-    def file_entry(ref, identifier)
-      ref["fileref"] or return
-      out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
-      out1 = @disambig.source2dest_filename(out)
-      ret = if ref["fileref"]
-              { type: "fileref", ref: @documents[Util::key identifier].file,
-                rel_path: ref["fileref"], url: ref["url"],
-                out_path: out1 } # @disambig.source2dest_filename(out) }
-            else { type: "id", ref: ref["id"] }
-            end
-      file_entry_copy(ref, ret)
-      ret.compact
-    end
-
-    def file_entry_copy(ref, ret)
-      %w(attachment sectionsplit index presentation-xml url
-         bare-after-first).each do |s|
-        ret[s.gsub("-", "").to_sym] = ref[s] if ref[s]
-      end
-    end
-
-    def add_document_suffix(identifier, doc)
-      document_suffix = Metanorma::Utils::to_ncname(identifier)
-      Metanorma::Utils::anchor_attributes.each do |(tag_name, attribute_name)|
-        Util::add_suffix_to_attributes(doc, document_suffix, tag_name,
-                                       attribute_name, @isodoc)
-      end
-      url_in_css_styles(doc, document_suffix)
-      doc.root["document_suffix"] ||= ""
-      doc.root["document_suffix"] += document_suffix
-    end
-
-    # update relative URLs, url(#...), in CSS in @style attrs (including SVG)
-    def url_in_css_styles(doc, document_suffix)
-      doc.xpath("//*[@style]").each do |s|
-        s["style"] = s["style"]
-          .gsub(%r{url\(#([^)]+)\)}, "url(#\\1_#{document_suffix})")
-      end
-    end
-
-    # return citation url for file
-    # @param doc [Boolean] I am a Metanorma document,
-    # so my URL should end with html or pdf or whatever
-    def url(ident, options)
-      data = get(ident)
-      data[:url] || targetfile(data, options)[1]
-    end
-
-    # are references to the file to be linked to a file in the collection,
-    # or externally? Determines whether file suffix anchors are to be used
-    def url?(ident)
-      data = get(ident) or return false
-      data[:url]
-    end
-
-    # return file contents + output filename for each file in the collection,
-    # given a docref entry
-    # @param data [Hash] docref entry
-    # @param read [Boolean] read the file in and return it
-    # @param doc [Boolean] I am a Metanorma document,
-    # so my URL should end with html or pdf or whatever
-    # @param relative [Boolean] Return output path,
-    # formed relative to YAML file, not input path, relative to calling function
-    # @return [Array<String, nil>]
-    def targetfile(data, options)
-      options = { read: false, doc: true, relative: false }.merge(options)
-      path = options[:relative] ? data[:rel_path] : data[:ref]
-      if data[:type] == "fileref"
-        ref_file path, data[:out_path], options[:read], options[:doc]
-      else
-        xml_file data[:id], options[:read]
-      end
-    end
-
-    def targetfile_id(ident, options)
-      targetfile(get(ident), options)
-    end
-
-    def ref_file(ref, out, read, doc)
-      file = File.read(ref, encoding: "utf-8") if read
-      filename = out.dup
-      filename.sub!(/\.xml$/, ".html") if doc
-      [file, filename]
-    end
-
-    def xml_file(id, read)
-      file = @xml.at(ns("//doc-container[@id = '#{id}']")).to_xml if read
-      filename = "#{id}.html"
-      [file, filename]
-    end
-
-    # map locality type and label (e.g. "clause" "1") to id = anchor for
-    # a document
-    # Note: will only key clauses, which have unambiguous reference label in
-    # locality. Notes, examples etc with containers are just plunked against
-    # UUIDs, so that their IDs can at least be registered to be tracked
-    # as existing.
-    def read_anchors(xml)
-      xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n,
-                                { locale: @locale })
-      xrefs.parse xml
-      xrefs.get.each_with_object({}) do |(k, v), ret|
-        read_anchors1(k, v, ret)
-      end
-    end
-
-    def read_anchors1(key, val, ret)
-      val[:type] ||= "clause"
-      ret[val[:type]] ||= {}
-      index = if val[:container] || val[:label].nil? || val[:label].empty?
-                UUIDTools::UUID.random_create.to_s
-              else val[:label]
-              end
-      ret[val[:type]][index] = key
-      ret[val[:type]][val[:value]] = key if val[:value]
-    end
-
-    # Also parse all ids in doc (including ones which won't be xref targets)
-    def read_ids(xml)
-      ret = {}
-      xml.traverse do |x|
-        x.text? and next
-        /^semantic__/.match?(x.name) and next
-        x["id"] and ret[x["id"]] = true
-      end
-      ret
-    end
-
-    def key(ident)
-      @c.decode(ident).gsub(/(\p{Zs})+/, " ").sub(/^metanorma-collection /, "")
-    end
-
-    def keys
-      @files.keys
-    end
-
-    def get(ident, attr = nil)
-      if attr then @files[key(ident)][attr]
-      else @files[key(ident)]
-      end
-    end
-
-    def set(ident, attr, value)
-      @files[key(ident)][attr] = value
-    end
-
-    def each
-      @files.each
-    end
-
-    def each_with_index
-      @files.each_with_index
-    end
-
-    def ns(xpath)
-      @isodoc.ns(xpath)
-    end
-  end
-end

data/lib/metanorma/files_lookup_sectionsplit.rb

@@ -1,84 +0,0 @@
-module Metanorma
-  # XML collection renderer
-  class FileLookup
-    def add_section_split
-      ret = @files.keys.each_with_object({}) do |k, m|
-        if @files[k][:sectionsplit] == "true" && !@files[k]["attachment"]
-          process_section_split_instance(k, m)
-          cleanup_section_split_instance(k, m)
-        end
-        m[k] = @files[k]
-      end
-      @files = ret
-    end
-
-    def process_section_split_instance(key, manifest)
-      s, sectionsplit_manifest = sectionsplit(@files[key][:ref],
-                                              @files[key][:out_path], key)
-      s.each_with_index do |f1, i|
-        add_section_split_instance(f1, manifest, key, i)
-      end
-      manifest["#{key}:index.html"] =
-        add_section_split_cover(sectionsplit_manifest, key)
-    end
-
-    def cleanup_section_split_instance(key, manifest)
-      @files_to_delete << manifest["#{key}:index.html"][:ref]
-      #@files[key].delete(:ids).delete(:anchors)
-      @files[key][:indirect_key] = @sectionsplit.key
-    end
-
-    def add_section_split_cover(manifest, ident)
-      cover = @sectionsplit
-        .section_split_cover(manifest, @parent.dir_name_cleanse(ident),
-                             one_doc_collection?)
-      @files[ident][:out_path] = cover
-      { attachment: true, index: false, out_path: cover,
-        ref: File.join(File.dirname(manifest.file), cover) }
-    end
-
-    def one_doc_collection?
-      return false
-      docs = 0
-      @files.each_value do |v|
-        v[:attachment] and next
-        v[:presentationxml] and next
-        docs += 1
-      end
-      docs > 1
-    end
-
-    def add_section_split_instance(file, manifest, key, idx)
-      presfile, newkey, xml =
-        add_section_split_instance_prep(file, key)
-      manifest[newkey] =
-        { parentid: key, presentationxml: true, type: "fileref",
-          rel_path: file[:url], out_path: File.basename(file[:url]),
-          anchors: read_anchors(xml), ids: read_ids(xml),
-          sectionsplit_output: true,
-          bibdata: @files[key][:bibdata], ref: presfile }
-      @files_to_delete << file[:url]
-      manifest[newkey][:bare] = true unless idx.zero?
-    end
-
-    def add_section_split_instance_prep(file, key)
-      presfile = File.join(File.dirname(@files[key][:ref]),
-                           File.basename(file[:url]))
-      newkey = key("#{key.strip} #{file[:title]}")
-      xml = Nokogiri::XML(File.read(presfile), &:huge)
-      [presfile, newkey, xml]
-    end
-
-    def sectionsplit(file, outfile, ident)
-      @sectionsplit = Sectionsplit
-        .new(input: file, base: outfile, dir: File.dirname(file),
-             output: outfile, compile_opts: @parent.compile_options,
-             fileslookup: self, ident: ident, isodoc: @isodoc)
-      coll = @sectionsplit.sectionsplit.sort_by { |f| f[:order] }
-      xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"), &:huge)
-      [coll, @sectionsplit
-        .collection_manifest(File.basename(file), coll, xml, nil,
-                             File.dirname(file))]
-    end
-  end
-end