metanorma 1.7.7 → 2.0.1

data/lib/metanorma/collection/document/document.rb

@@ -0,0 +1,133 @@
+module Metanorma
+  class Collection
+    class Document
+      # @return [Strin]
+      attr_reader :file, :attachment, :bibitem, :index
+
+      # @param bibitem [RelatonBib::BibliographicItem]
+      def initialize(bibitem, file, options = {})
+        @bibitem = bibitem
+        @file = file
+        @attachment = options[:attachment]
+        @index = options[:index]
+        @index = true if @index.nil?
+        @raw = options[:raw]
+      end
+
+      class << self
+        # @param file [String] file path
+        # @param attachment [Bool] is an attachment
+        # @param identifier [String] is the identifier assigned the file
+        # in the collection file
+        # @param index [Bool] is indication on whether to index this file in coverpage
+        # @return [Metanorma::Document]
+        def parse_file(file, attachment, identifier = nil, index = true)
+          new(bibitem(file, attachment, identifier), file,
+              { attachment: attachment, index: index })
+        end
+
+        # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
+        # @return [Metanorma::Document]
+        def parse_xml(xml)
+          new from_xml(xml)
+        end
+
+        # raw XML file, can be used to put in entire file instead of just bibitem
+        def raw_file(filename)
+          doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8"), &:huge)
+          new(doc, filename, raw: true)
+        end
+
+        def attachment_bibitem(identifier)
+          Nokogiri::XML <<~DOCUMENT
+            <bibdata><docidentifier>#{identifier}</docidentifier></bibdata>
+          DOCUMENT
+        end
+
+        private
+
+        def mn2relaton_parser(tag)
+          case tag.sub(/-standard/, "")
+          when "bipm" then RelatonBipm::XMLParser
+          when "bsi" then RelatonBsi::XMLParser
+          when "ietf" then RelatonIetf::XMLParser
+          when "iho" then RelatonIho::XMLParser
+          when "itu" then RelatonItu::XMLParser
+          when "iec" then RelatonIec::XMLParser
+          when "iso" then RelatonIsoBib::XMLParser
+          when "nist" then RelatonNist::XMLParser
+          when "ogc" then RelatonOgc::XMLParser
+          else RelatonBib::XMLParser
+          end
+        end
+
+        # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
+        # @return [RelatonBib::BibliographicItem,RelatonIso::IsoBibliographicItem]
+        def from_xml(xml)
+          b = xml.at("//xmlns:bibitem|//xmlns:bibdata")
+          r = mn2relaton_parser(xml.root.name)
+          r.from_xml(b.to_xml)
+        end
+
+        # @param file [String]
+        # @return [Symbol] file type
+        def format(file)
+          case file
+          when /\.xml$/ then :xml
+          when /.ya?ml$/ then :yaml
+          end
+        end
+
+        # @param file [String]
+        # @return [RelatonBib::BibliographicItem,
+        #   RelatonIso::IsoBibliographicItem]
+        def bibitem(file, attachment, identifier)
+          if attachment then attachment_bibitem(identifier)
+          else
+            case format(file)
+            when :xml
+              from_xml (Nokogiri::XML(File.read(file, encoding: "UTF-8"),
+                                      &:huge))
+            when :yaml
+              yaml = File.read(file, encoding: "UTF-8")
+              Relaton::Cli::YAMLConvertor.convert_single_file(yaml)
+            end
+          end
+        end
+      end
+
+      # @param builder [Nokogiri::XML::Builder, nil]
+      # @return [Nokogiri::XML::Builder, String]
+      def to_xml(builder = nil)
+        if builder
+          render_xml builder
+        else
+          Nokogiri::XML::Builder.new do |b|
+            root = render_xml b
+            root["xmlns"] = "http://metanorma.org"
+          end.to_xml
+        end
+      end
+
+      # @return [String]
+      def type
+        first = @bibitem.docidentifier.first
+        @type ||= (first&.type&.downcase ||
+                   first&.id&.match(/^[^\s]+/)&.to_s)&.downcase ||
+          "standoc"
+      end
+
+      private
+
+      def render_xml(builder)
+        if @raw
+          builder.parent.add_child(@bibitem.root)
+        else
+          builder.send("#{type}-standard") do |b|
+            b << @bibitem.to_xml(bibdata: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/collection/filelookup/filelookup.rb

@@ -0,0 +1,250 @@
+require "isodoc"
+require "htmlentities"
+require "metanorma-utils"
+require_relative "filelookup_sectionsplit"
+
+module Metanorma
+  class Collection
+    class FileLookup
+      attr_accessor :files_to_delete, :parent
+
+      # hash for each document in collection of document identifier to:
+      # document reference (fileref or id), type of document reference,
+      # and bibdata entry for that file
+      # @param path [String] path to collection
+      def initialize(path, parent)
+        @c = HTMLEntities.new
+        @files = {}
+        @parent = parent
+        @xml = parent.xml
+        @isodoc = parent.isodoc
+        @path = path
+        @compile = parent.compile
+        @documents = parent.documents
+        @files_to_delete = []
+        @disambig = Util::DisambigFiles.new
+        @manifest = parent.manifest
+        read_files(@manifest.entry)
+      end
+
+      def read_files(entries)
+        Array(entries).each do |e|
+          e.file and read_file(e)
+          read_files(e.entry)
+        end
+      end
+
+      def read_file(manifest)
+        i, k = read_file_idents(manifest)
+        entry = file_entry(manifest, k) or return
+        bibdata_process(entry, i)
+        bibitem_process(entry)
+        @files[key(i)] = entry
+      end
+
+      def read_file_idents(manifest)
+        id = manifest.identifier
+        sanitised_id = key(@isodoc.docid_prefix("", manifest.identifier.dup))
+        #       if manifest.bibdata and # NO, DO NOT FISH FOR THE GENUINE IDENTIFIER IN BIBDATA
+        #         d = manifest.bibdata.docidentifier.detect { |x| x.primary } ||
+        #           manifest.bibdata.docidentifier.first
+        #         k = d.id
+        #         i = key(@isodoc.docid_prefix(d.type, d.id.dup))
+        #       end
+        [id, sanitised_id]
+      end
+
+      def bibdata_process(entry, ident)
+        if entry[:attachment]
+          entry[:bibdata] =
+            Metanorma::Collection::Document.attachment_bibitem(ident).root
+        else
+          file, _filename = targetfile(entry, read: true)
+          xml = Nokogiri::XML(file, &:huge)
+          add_document_suffix(ident, xml)
+          entry.merge!(anchors: read_anchors(xml), ids: read_ids(xml),
+                       bibdata: xml.at(ns("//bibdata")),
+                       document_suffix: xml.root["document_suffix"])
+        end
+      end
+
+      def bibitem_process(entry)
+        entry[:bibitem] = entry[:bibdata].dup
+        entry[:bibitem].name = "bibitem"
+        entry[:bibitem]["hidden"] = "true"
+        entry[:bibitem].at("./*[local-name() = 'ext']")&.remove
+      end
+
+      # ref is the absolute source file address
+      # rel_path is the relative source file address, relative to the YAML locaton
+      # out_path is the destination file address, with any references outside
+      # the working directory (../../...) truncated, and based on relative path
+      # identifier is the id with only spaces, no nbsp
+      def file_entry(ref, identifier)
+        ref.file or return
+        abs = @documents[Util::key identifier].file
+        ret = if ref.file
+                { type: "fileref", ref: abs, rel_path: ref.file, url: ref.url,
+                  out_path: output_file_path(ref) }
+              else { type: "id", ref: ref.id }
+              end
+        file_entry_copy(ref, ret)
+        ret.compact
+      end
+
+      # TODO make the output file location reflect source location universally,
+      # not just for attachments: no File.basename
+      def output_file_path(ref)
+        f = File.basename(ref.file)
+        ref.attachment and f = ref.file
+        @disambig.source2dest_filename(f)
+      end
+
+      def file_entry_copy(ref, ret)
+        %w(attachment sectionsplit index presentation-xml url
+           bare-after-first).each do |s|
+          ref.respond_to?(s.to_sym) and
+            ret[s.gsub("-", "").to_sym] = ref.send(s)
+        end
+      end
+
+      def add_document_suffix(identifier, doc)
+        document_suffix = Metanorma::Utils::to_ncname(identifier)
+        Metanorma::Utils::anchor_attributes.each do |(tag_name, attribute_name)|
+          Util::add_suffix_to_attributes(doc, document_suffix, tag_name,
+                                         attribute_name, @isodoc)
+        end
+        url_in_css_styles(doc, document_suffix)
+        doc.root["document_suffix"] ||= ""
+        doc.root["document_suffix"] += document_suffix
+      end
+
+      # update relative URLs, url(#...), in CSS in @style attrs (including SVG)
+      def url_in_css_styles(doc, document_suffix)
+        doc.xpath("//*[@style]").each do |s|
+          s["style"] = s["style"]
+            .gsub(%r{url\(#([^)]+)\)}, "url(#\\1_#{document_suffix})")
+        end
+      end
+
+      # return citation url for file
+      # @param doc [Boolean] I am a Metanorma document,
+      # so my URL should end with html or pdf or whatever
+      def url(ident, options)
+        data = get(ident)
+        data[:url] || targetfile(data, options)[1]
+      end
+
+      # are references to the file to be linked to a file in the collection,
+      # or externally? Determines whether file suffix anchors are to be used
+      def url?(ident)
+        data = get(ident) or return false
+        data[:url]
+      end
+
+      # return file contents + output filename for each file in the collection,
+      # given a docref entry
+      # @param data [Hash] docref entry
+      # @param read [Boolean] read the file in and return it
+      # @param doc [Boolean] I am a Metanorma document,
+      # so my URL should end with html or pdf or whatever
+      # @param relative [Boolean] Return output path,
+      # formed relative to YAML file, not input path, relative to calling function
+      # @return [Array<String, nil>]
+      def targetfile(data, options)
+        options = { read: false, doc: true, relative: false }.merge(options)
+        path = options[:relative] ? data[:rel_path] : data[:ref]
+        if data[:type] == "fileref"
+          ref_file path, data[:out_path], options[:read], options[:doc]
+        else
+          xml_file data[:id], options[:read]
+        end
+      end
+
+      def targetfile_id(ident, options)
+        targetfile(get(ident), options)
+      end
+
+      def ref_file(ref, out, read, doc)
+        file = File.read(ref, encoding: "utf-8") if read
+        filename = out.dup
+        filename.sub!(/\.xml$/, ".html") if doc
+        [file, filename]
+      end
+
+      def xml_file(id, read)
+        file = @xml.at(ns("//doc-container[@id = '#{id}']")).to_xml if read
+        filename = "#{id}.html"
+        [file, filename]
+      end
+
+      # map locality type and label (e.g. "clause" "1") to id = anchor for
+      # a document
+      # Note: will only key clauses, which have unambiguous reference label in
+      # locality. Notes, examples etc with containers are just plunked against
+      # UUIDs, so that their IDs can at least be registered to be tracked
+      # as existing.
+      def read_anchors(xml)
+        xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n,
+                                  { locale: @locale })
+        xrefs.parse xml
+        xrefs.get.each_with_object({}) do |(k, v), ret|
+          read_anchors1(k, v, ret)
+        end
+      end
+
+      def read_anchors1(key, val, ret)
+        val[:type] ||= "clause"
+        ret[val[:type]] ||= {}
+        index = if val[:container] || val[:label].nil? || val[:label].empty?
+                  UUIDTools::UUID.random_create.to_s
+                else val[:label]
+                end
+        ret[val[:type]][index] = key
+        ret[val[:type]][val[:value]] = key if val[:value]
+      end
+
+      # Also parse all ids in doc (including ones which won't be xref targets)
+      def read_ids(xml)
+        ret = {}
+        xml.traverse do |x|
+          x.text? and next
+          /^semantic__/.match?(x.name) and next
+          x["id"] and ret[x["id"]] = true
+        end
+        ret
+      end
+
+      def key(ident)
+        @c.decode(ident).gsub(/(\p{Zs})+/, " ").sub(/^metanorma-collection /,
+                                                    "")
+      end
+
+      def keys
+        @files.keys
+      end
+
+      def get(ident, attr = nil)
+        if attr then @files[key(ident)][attr]
+        else @files[key(ident)]
+        end
+      end
+
+      def set(ident, attr, value)
+        @files[key(ident)][attr] = value
+      end
+
+      def each
+        @files.each
+      end
+
+      def each_with_index
+        @files.each_with_index
+      end
+
+      def ns(xpath)
+        @isodoc.ns(xpath)
+      end
+    end
+  end
+end

data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb

@@ -0,0 +1,87 @@
+require_relative "../sectionsplit/sectionsplit"
+
+module Metanorma
+  class Collection
+    class FileLookup
+      def add_section_split
+        ret = @files.keys.each_with_object({}) do |k, m|
+          if @files[k][:sectionsplit] && !@files[k][:attachment]
+            process_section_split_instance(k, m)
+            cleanup_section_split_instance(k, m)
+          end
+          m[k] = @files[k]
+        end
+        @files = ret
+      end
+
+      def process_section_split_instance(key, manifest)
+        s, sectionsplit_manifest = sectionsplit(key)
+        s.each_with_index do |f1, i|
+          add_section_split_instance(f1, manifest, key, i)
+        end
+        manifest["#{key}:index.html"] =
+          add_section_split_cover(sectionsplit_manifest, key)
+      end
+
+      def cleanup_section_split_instance(key, manifest)
+        @files_to_delete << manifest["#{key}:index.html"][:ref]
+        # @files[key].delete(:ids).delete(:anchors)
+        @files[key][:indirect_key] = @sectionsplit.key
+      end
+
+      def add_section_split_cover(manifest, ident)
+        cover = @sectionsplit
+          .section_split_cover(manifest, @parent.dir_name_cleanse(ident),
+                               one_doc_collection?)
+        @files[ident][:out_path] = cover
+        { attachment: true, index: false, out_path: cover,
+          ref: File.join(File.dirname(manifest.file), cover) }
+      end
+
+      def one_doc_collection?
+        return false
+        docs = 0
+        @files.each_value do |v|
+          v[:attachment] and next
+          v[:presentationxml] and next
+          docs += 1
+        end
+        docs > 1
+      end
+
+      def add_section_split_instance(file, manifest, key, idx)
+        presfile, newkey, xml =
+          add_section_split_instance_prep(file, key)
+        manifest[newkey] =
+          { parentid: key, presentationxml: true, type: "fileref",
+            rel_path: file[:url], out_path: File.basename(file[:url]),
+            anchors: read_anchors(xml), ids: read_ids(xml),
+            sectionsplit_output: true,
+            bibdata: @files[key][:bibdata], ref: presfile }
+        @files_to_delete << file[:url]
+        manifest[newkey][:bare] = true unless idx.zero?
+      end
+
+      def add_section_split_instance_prep(file, key)
+        presfile = File.join(File.dirname(@files[key][:ref]),
+                             File.basename(file[:url]))
+        newkey = key("#{key.strip} #{file[:title]}")
+        xml = Nokogiri::XML(File.read(presfile), &:huge)
+        [presfile, newkey, xml]
+      end
+
+      def sectionsplit(ident)
+        file = @files[ident][:ref]
+        @sectionsplit = ::Metanorma::Collection::Sectionsplit
+          .new(input: file, base: @files[ident][:out_path], dir: File.dirname(file),
+               output: @files[ident][:out_path], compile_opts: @parent.compile_options,
+               fileslookup: self, ident: ident, isodoc: @isodoc)
+        coll = @sectionsplit.sectionsplit.sort_by { |f| f[:order] }
+        xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"), &:huge)
+        [coll, @sectionsplit
+          .collection_manifest(File.basename(file), coll, xml, nil,
+                               File.dirname(file))]
+      end
+    end
+  end
+end

data/lib/metanorma/collection/manifest/manifest.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require_relative "../../util/util"
+
+module Metanorma
+  class Collection
+    class Manifest
+      # @return [Metanorma::Collection]
+      attr_reader :collection, :config
+
+      # @param level [String]
+      # @param dir [String]
+      # @param title [String, nil]
+      # @param docref [Array<Hash{String=>String}>]
+      # @param manifest [Array<Metanorma::Collection::Manifest>]
+      def initialize(config, collection, dir)
+        @collection = collection
+        @dir = dir
+        @disambig = ::Metanorma::Collection::Util::DisambigFiles.new
+        @config = manifest_postprocess(config)
+      end
+
+      def manifest_postprocess(config)
+        manifest_bibdata(config)
+        manifest_expand_yaml(config, @dir)
+        manifest_compile_adoc(config)
+        manifest_filexist(config)
+        manifest_sectionsplit(config)
+        manifest_identifier(config)
+        config
+      end
+
+      def manifest_bibdata(config)
+        b = config.bibdata
+        @lang = b&.language&.first || "en"
+        @script = b&.script&.first || "Latn"
+      end
+
+      GUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+
+      def manifest_identifier(config)
+        no_id = populate_id_from_doc(config)
+        config.identifier =
+          if no_id && config.file # make file name be id
+            @collection.class.resolve_identifier(File.basename(config.file))
+          else
+            @collection.class.resolve_identifier(config.identifier)
+          end
+        Array(config.entry).each do |f|
+          manifest_identifier(f)
+        end
+      end
+
+      def populate_id_from_doc(config)
+        no_id = /^#{GUID}$/o.match?(config.identifier)
+        # GUID assumed to be no identifier supplied
+        if no_id && /\.xml$/.match?(config.file) &&
+            (i = retrieve_id_from_doc(config.file))
+          config.identifier = i
+          no_id = false
+        end
+        no_id
+      end
+
+      def retrieve_id_from_doc(file)
+        x = Nokogiri::XML(File.read(File.join(@dir, file)), &:huge)
+        i = x.at("//xmlns:bibdata/xmlns:docidentifier[@primary = 'true']") ||
+          x.at("//xmlns:bibdata/xmlns:docidentifier")
+        i or return nil
+        @doctype ||= i["type"]&.downcase || "standoc"
+        load_isodoc
+        Util::key(@isodoc.docid_prefix(i["type"], i.text))
+      end
+
+      def load_isodoc
+        @isodoc and return
+        @collection.compile.load_flavor(@doctype)
+        @isodoc = Util::load_isodoc(@doctype)
+        @isodoc.i18n_init(@lang, @script, nil) # for @i18n.all_parts in docid
+      end
+
+      def manifest_sectionsplit(config)
+        if config.sectionsplit && !config.file
+          config.sectionsplit = nil
+          Array(config.entry).each do |e|
+            e.attachment and next
+            e.sectionsplit = true
+          end
+        end
+        Array(config.entry).each do |f|
+          manifest_sectionsplit(f)
+        end
+      end
+
+      def manifest_filexist(config)
+        if config.file
+          file = @collection.class.resolve_fileref(@dir, config.file)
+          @collection.class.check_file_existence(file)
+          config.file = Pathname.new(file).relative_path_from(Pathname.new(@dir))
+        end
+        Array(config.entry).each do |f|
+          manifest_filexist(f)
+        end
+      end
+
+      def manifest_expand_yaml(config, dir)
+        Array(config.entry).each do |e|
+          currdir = dir
+          /\.ya?ml$/.match?(e.file) and
+            currdir = manifest_expand_yaml_entry(e, dir)
+          manifest_expand_yaml(e, currdir)
+        end
+        config
+      end
+
+      def manifest_expand_yaml_entry(entry, dir)
+        f = @collection.class.resolve_fileref(dir, entry.file)
+        currdir = File.dirname(f)
+        @collection.class.check_file_existence(f)
+        entry.file = nil
+        entry.entry = ::Metanorma::Collection::Config::Config.from_yaml(File.read(f)).manifest
+        if currdir != dir
+          prefix = Pathname.new(currdir).relative_path_from(Pathname.new(dir))
+          update_filepaths(entry.entry, prefix.to_s)
+        end
+        currdir
+      end
+
+      def update_filepaths(entry, prefix)
+        entry.file && !(Pathname.new entry.file).absolute? and
+          entry.file = File.join(prefix, entry.file)
+        entry.entry.each do |f|
+          update_filepaths(f, prefix)
+        end
+      end
+
+      def manifest_compile_adoc(config)
+        if /\.adoc$/.match?(config.file)
+          file = @collection.class.resolve_fileref(@dir, config.file)
+          config.file = compile_adoc(file, config.file)
+        end
+        Array(config.entry).each do |f|
+          manifest_compile_adoc(f)
+        end
+      end
+
+      def compile_adoc(resolved_filename, rel_filename)
+        compile_adoc_file(resolved_filename)
+        set_adoc2xml(rel_filename)
+      end
+
+      # @param fileref [String]
+      def set_adoc2xml(fileref)
+        File.join(
+          File.dirname(fileref),
+          File.basename(fileref).gsub(/.adoc$/, ".xml"),
+        )
+      end
+
+      # param filepath [String]
+      # @raise [AdocFileNotFoundException]
+      def compile_adoc_file(file)
+        f = (Pathname.new file).absolute? ? file : File.join(@dir, file)
+        unless File.exist? f
+          raise AdocFileNotFoundException.new "#{f} not found!"
+        end
+
+        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...", :info)
+        ::Metanorma::Compile.new
+          .compile(f, agree_to_terms: true, no_install_fonts: true,
+                      extension_keys: [:xml])
+        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...done!", :info)
+      end
+
+      def documents(mnf = @config)
+        Array(mnf.entry).each_with_object({}) do |dr, m|
+          if dr.file
+            m[Util::key dr.identifier] = documents_add(dr)
+          elsif dr.entry
+            m.merge! documents(dr)
+          end
+          m
+        end
+      end
+
+      def documents_add(docref)
+        ::Metanorma::Collection::Document.parse_file(
+          Util::rel_path_resolve(@dir, docref.file),
+          docref.attachment, docref.identifier, docref.index
+        )
+      end
+
+      def to_xml(builder = nil)
+        clean_manifest(@config)
+        if builder
+          builder.parent.add_child(@config.to_xml)
+        else
+          @config.to_xml
+        end
+      end
+
+      def clean_manifest_bibdata(mnf)
+        if mnf.file && !mnf.attachment && !mnf.sectionsplit && @collection &&
+            d = @collection.bibdatas[Util::key mnf.identifier]
+          mnf.bibdata = d.bibitem.dup
+        end
+      end
+
+      def clean_manifest_id(mnf)
+        @collection.directives.detect do |d|
+          d.key == "documents-inline"
+        end or return
+        id = @collection.documents.find_index do |k, _|
+          k == mnf.identifier
+        end
+        id and mnf.id = format("doc%<index>09d", index: id)
+      end
+
+      def clean_manifest(mnf)
+        clean_manifest_bibdata(mnf)
+        # mnf.file &&= @disambig.strip_root(mnf.file)
+        clean_manifest_id(mnf)
+        Array(mnf.entry).each { |e| clean_manifest(e) }
+      end
+
+      # @return [Array<Hash{String=>String}>]
+      def docrefs
+        @config.entry
+      end
+
+      def docref_by_id(docid)
+        @config.entry.detect { |k| k.identifier == docid } ||
+          @config.entry.detect { |k| /^#{k.identifier}/ =~ docid }
+      end
+    end
+  end
+end