metanorma 2.1.4 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59bdb97eff1bbdd231593855917d6ecc44e723dfc6bdec7bdd84a739a54d6162
-  data.tar.gz: ed4c015ceb9003f8aa9d8151cae07d59ef8d645f1812bac237706994ad707402
+  metadata.gz: af06733dd8691111633e6309c85ebfa069baa2edf16cbd7e8458a17ad34d4b2a
+  data.tar.gz: e924bb9b298cc37c2cebef6f8d9251ae3979e9cd9b537539057a5ade07e11c88
 SHA512:
-  metadata.gz: 1bd7eb1ec3a9765a3135f35b5f9011463716546c6d86d1d487e99cbef2f97147ef29f685694e03945a7fac26c25c62ec580f5657b76ecda50d1054282dfb5896
-  data.tar.gz: abb72caf7d58a9f24eafa7c5bf6fa1fd8ee2cfe2333d30d18d83fc4b1c13245dc7bf79bc60f6c7cd77654e0ec787ca1a44f0d2844eb08e75af5c4489b47096df
+  metadata.gz: 29499feb91dd56f4fe6f281f885263c897fb555a9905ce3a32b56d175199b520e9066041ff96489d5221302c259a08bf0d3c07bde19a232b3893d79a0bde747b
+  data.tar.gz: 717f1409b88dfc565393c71cddb0fda604e062d38e80b45c470dc06957ac5e7cf2f5adfd7ef7083666aeb4f7f7b6356fcbc95014bb2e9091c9c927232d7c225f

data/.rubocop.yml

@@ -7,4 +7,7 @@ inherit_from:
 # ...
 
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 3.1
+
+Lint/MissingSuper:
+  AllowedParentClasses: [Liquid::Drop]

data/lib/metanorma/collection/collection.rb

@@ -92,15 +92,22 @@ module Metanorma
     end
 
     def render(opts)
-      if opts[:format].nil? || opts[:format].empty?
+      opts[:format].nil? || opts[:format].empty? and
         opts[:format] = @format || [:html]
-      end
       opts[:log] = @log
       opts[:flavor] = @flavor
+      output_folder(opts)
       ::Metanorma::Collection::Renderer.render self, opts
       clean_exit
     end
 
+    def output_folder(opts)
+      opts[:output_folder] ||= config.output_folder
+      opts[:output_folder] && !Pathname.new(opts[:output_folder]).absolute? and
+        opts[:output_folder] = File.join(@dirname, opts[:output_folder])
+      warn opts[:output_folder]
+    end
+
     # @return [String, nil]
     attr_reader :prefatory, :final
 
@@ -165,7 +172,7 @@ module Metanorma
     def fetch_flavor
       docid = @bibdata.docidentifier.first or return
       f = docid.type.downcase || docid.id.sub(/\s.*$/, "").downcase or return
-      require ::Metanorma::Compile.new.stdtype2flavor(f)
+      require ::Metanorma::Compile.new.stdtype2flavor_gem(f)
       f
     rescue LoadError
       nil
@@ -216,7 +223,6 @@ module Metanorma
       def resolve_fileref(ref_folder, fileref)
         warn ref_folder
         warn fileref
-        #require "debug"; binding.b
         unless @fileref_resolver
           (Pathname.new fileref).absolute? or
             fileref = File.join(ref_folder, fileref)

data/lib/metanorma/collection/config/config.rb

@@ -124,6 +124,27 @@ module Metanorma
           end
         end
 
+        def self.flavor_from_yaml(yaml)
+          yaml["directives"]&.detect do |x|
+            x.is_a?(Hash) && x.has_key?("flavor")
+          end&.dig("flavor")&.upcase
+        end
+
+        def self.flavor_to_bibdata(file)
+          # propagate flavor from directives to bibdata
+          yaml = YAML.safe_load(file)
+          flavor = flavor_from_yaml(yaml) or return file
+          yaml["bibdata"] or return file
+          yaml["bibdata"]["ext"] ||= {}
+          yaml["bibdata"]["ext"]["flavor"] ||= flavor
+          yaml.to_yaml
+        end
+
+        def self.from_yaml(file)
+          file = flavor_to_bibdata(file)
+          super
+        end
+
         include Converters
       end
     end

data/lib/metanorma/collection/config/manifest.rb

@@ -13,7 +13,7 @@ module Metanorma
 
       class Manifest < ::Lutaml::Model::Serializable
         attribute :identifier, :string,
-          default: -> { UUIDTools::UUID.random_create.to_s }
+                  default: -> { UUIDTools::UUID.random_create.to_s }
         attribute :id, :string
         attribute :bibdata, Bibdata
         attribute :type, :string
@@ -25,6 +25,7 @@ module Metanorma
         attribute :index, :boolean, default: -> { true }
         attribute :entry, Manifest, collection: true
         attribute :file, :string
+        attribute :format, :string, collection: true
 
         yaml do
           map "identifier", to: :identifier
@@ -46,6 +47,7 @@ module Metanorma
                           with: { from: :docref_from_yaml, to: :nop_to_yaml }
           map "bibdata", to: :bibdata, with: { from: :bibdata_from_yaml,
                                                to: :bibdata_to_yaml }
+          map "format", to: :format, render_default: true
         end
 
         xml do
@@ -59,6 +61,7 @@ module Metanorma
           map_element "identifier", to: :identifier, render_default: true
           map_element "type", to: :type
           map_element "title", to: :title
+          map_element "format", to: :format
           map_element "bibdata", to: :bibdata, with: { from: :bibdata_from_xml,
                                                        to: :bibdata_to_xml }
           map_element "entry", to: :entry

data/lib/metanorma/collection/document/document.rb

@@ -68,6 +68,8 @@ module Metanorma
         def from_xml(xml)
           b = xml.at("//xmlns:bibitem|//xmlns:bibdata")
           r = mn2relaton_parser(xml.root.name)
+          # Relaton doesn't understand Pres XML tags
+          b.xpath("//xmlns:fmt-identifier").each(&:remove)
           r.from_xml(b.to_xml)
         end
 

data/lib/metanorma/collection/filelookup/filelookup.rb

@@ -18,22 +18,31 @@ module Metanorma
         @parent = parent
         @xml = parent.xml
         @isodoc = parent.isodoc
+        @isodoc_presxml = parent.isodoc_presxml
         @path = path
         @compile = parent.compile
         @documents = parent.documents
         @files_to_delete = []
         @disambig = Util::DisambigFiles.new
         @manifest = parent.manifest
-        read_files(@manifest.entry)
+        read_files(@manifest.entry, parent.manifest)
+        #warn pp @files
       end
 
-      def read_files(entries)
+      def read_files(entries, parent)
         Array(entries).each do |e|
+          derive_format(e, parent)
           e.file and read_file(e)
-          read_files(e.entry)
+          read_files(e.entry, e)
         end
       end
 
+      def derive_format(entry, parent)
+        entry.attachment and return
+        entry.format ||= parent.format || %w(xml presentation html)
+        entry.format |= ["xml", "presentation"]
+      end
+
       def read_file(manifest)
         i, k = read_file_idents(manifest)
         entry = file_entry(manifest, k) or return
@@ -97,7 +106,8 @@ module Metanorma
         abs = @documents[Util::key identifier].file
         ret = if ref.file
                 { type: "fileref", ref: abs, rel_path: ref.file, url: ref.url,
-                  out_path: output_file_path(ref) }
+                  out_path: output_file_path(ref),
+                  format: ref.format&.map(&:to_sym) }.compact
               else { type: "id", ref: ref.id }
               end
         file_entry_copy(ref, ret)
@@ -122,7 +132,7 @@ module Metanorma
 
       def add_document_suffix(identifier, doc)
         document_suffix = Metanorma::Utils::to_ncname(identifier)
-        Metanorma::Utils::anchor_attributes.each do |(tag_name, attr_name)|
+        Util::anchor_id_attributes.each do |(tag_name, attr_name)|
           Util::add_suffix_to_attrs(doc, document_suffix, tag_name, attr_name,
                                     @isodoc)
         end
@@ -221,7 +231,6 @@ module Metanorma
         ret = {}
         xml.traverse do |x|
           x.text? and next
-          /^semantic__/.match?(x.name) and next
           x["id"] and ret[x["id"]] = true
         end
         ret

data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb

@@ -89,7 +89,7 @@ module Metanorma
         m = { parentid: key, presentationxml: true, type: "fileref",
               rel_path: file[:url], out_path: File.basename(file[:url]),
               anchors: anchors, anchors_lookup: anchors_lookup(anchors),
-              ids: read_ids(xml),
+              ids: read_ids(xml), format: @files[key][:format],
               sectionsplit_output: true, indirect_key: @sectionsplit.key,
               bibdata: @files[key][:bibdata], ref: presfile }
         m[:bare] = true unless idx.zero?
@@ -110,8 +110,9 @@ module Metanorma
         @sectionsplit = ::Metanorma::Collection::Sectionsplit
           .new(input: file, base: @files[ident][:out_path],
                dir: File.dirname(file), output: @files[ident][:out_path],
-               compile_opts: @parent.compile_options,
-               fileslookup: self, ident: ident, isodoc: @isodoc,
+               compile_opts: @parent.compile_options, ident: ident,
+               fileslookup: self, isodoc: @isodoc,
+               isodoc_presxml: @isodoc_presxml,
                document_suffix: @files[ident][:document_suffix])
         coll = @sectionsplit.sectionsplit.sort_by { |f| f[:order] }
         xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"), &:huge)

data/lib/metanorma/collection/manifest/manifest.rb

@@ -96,7 +96,6 @@ module Metanorma
 
       def manifest_filexist(config)
         if config.file
-        #require "debug"; binding.b
           file = @collection.class.resolve_fileref(@dir, config.file)
           @collection.class.check_file_existence(file)
           config.file = Pathname.new(file).relative_path_from(Pathname.new(@dir))
@@ -111,7 +110,6 @@ module Metanorma
           currdir = dir
           /\.ya?ml$/.match?(e.file) and
             currdir = manifest_expand_yaml_entry(e, dir)
-          #require "debug"; binding.b
           manifest_expand_yaml(e, currdir)
         end
         config
@@ -122,10 +120,8 @@ module Metanorma
         currdir = File.dirname(f)
         @collection.class.check_file_existence(f)
         entry.file = nil
-        #require 'debug'; binding.b
         entry.entry = ::Metanorma::Collection::Config::Config.from_yaml(File.read(f)).manifest
         if currdir != dir
-          #require "debug"; binding.b
           prefix = Pathname.new(currdir).relative_path_from(Pathname.new(dir))
           update_filepaths(entry.entry, prefix.to_s)
         end
@@ -133,10 +129,9 @@ module Metanorma
       end
 
       def update_filepaths(entry, prefix)
-        #require "debug"; binding.b
         entry.file && !(Pathname.new entry.file).absolute? and
           entry.file = File.join(prefix, entry.file)
-        entry.entry.each do |f|
+        entry&.entry&.each do |f|
           update_filepaths(f, prefix)
         end
       end
@@ -170,18 +165,19 @@ module Metanorma
         f = (Pathname.new file).absolute? ? file : File.join(@dir, file)
         File.exist?(f) or raise AdocFileNotFoundException.new "#{f} not found!"
         compile_adoc_file?(file) or return
-        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...", :info)
+        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...", :warning)
         ::Metanorma::Compile.new
           .compile(f, agree_to_terms: true, install_fonts: false,
                       extension_keys: [:xml])
-        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...done!", :info)
+        ::Metanorma::Util.log("[metanorma] Info: Compiling #{f}...done!", :warning)
       end
 
       def compile_adoc_file?(file)
+        !File.exist?(file.sub(/\.adoc$/, ".xml")) and return true
         @collection.directives.detect do |d|
-          d.key == "recompile-xml"
-        end and return true
-        !File.exist?(file.sub(/\.adoc$/, ".xml"))
+          d.key == "recompile-xml" && d.value == "false"
+        end and return false
+        true
       end
 
       def documents(mnf = @config)

data/lib/metanorma/collection/renderer/fileparse.rb

@@ -16,8 +16,6 @@ module Metanorma
       # @return [String] XML content
       def update_xrefs(file, docid, internal_refs)
         xml, sso = update_xrefs_prep(file, docid)
-        #require "debug"; binding.b if /This document is also unrelated/.match?(xml.to_xml)
-        #warn (/fmt-title/.match?(xml.to_xml) ? "*** PRESENTATION" : "*** SEMANTIC")
         @nested || sso or
           Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
                                                            @isodoc, sso)
@@ -26,7 +24,7 @@ module Metanorma
         @files.add_document_suffix(docid, xml)
         @nested or update_sectionsplit_refs_to_docs(xml, internal_refs, sso)
         update_direct_refs_to_docs(xml, docid, sso)
-        hide_refs(xml)
+        ::Metanorma::Collection::Util::hide_refs(xml)
         sso and eref2link(xml, sso)
         @nested or svgmap_resolve(xml, docid, sso)
         xml.to_xml
@@ -109,7 +107,6 @@ module Metanorma
             else no_anchor[k] << e end
           end
         end
-        #require "debug"; binding.b
         [erefs, no_anchor, anchors, Util::gather_bibitemids(docxml, presxml)]
       end
 
@@ -128,8 +125,9 @@ module Metanorma
 
       # Resolve erefs to a container of ids in another doc,
       # to an anchor eref (direct link)
-      def update_indirect_refs_to_docs(docxml, _docidentifier, internal_refs, presxml)
-        bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml, presxml)
+      def update_indirect_refs_to_docs(docxml, _docid, internal_refs, presxml)
+        bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml,
+                                                                        presxml)
         internal_refs.each do |schema, ids|
           add_suffix = doc_suffix && doc_type && doc_type != schema
           ids.each do |id, file|
@@ -149,37 +147,15 @@ module Metanorma
          docxml.root["document_suffix"], docxml.root["type"], {}]
       end
 
-      # KILL
-      def indirect_ref_key(schema, id, doc_suffix, doc_type)
-        /^#{schema}_/.match?(id) and return id
-        key = [schema, id, doc_suffix, doc_type].join("::")
-        x = @indirect_keys[key] and return x
-        ret = "#{schema}_#{id}"
-        doc_suffix && doc_type && doc_type != schema and
-          ret = "#{ret}_#{doc_suffix}"
-        @indirect_keys[key] = ret
-        ret
-      end
-
       def indirect_ref_key(schema, id, doc_suffix, add_suffix)
         /^#{schema}_/.match?(id) and return id
-        #key = "#{schema}_#{id}"
         x = @indirect_keys.dig(schema, id) and return x
         @indirect_keys[schema] ||= {}
         @indirect_keys[schema][id] = if add_suffix
-                                  "#{schema}_#{id}_#{doc_suffix}"
-                                else
-                                   "#{schema}_#{id}"
-                                end
-      end
-
-      # KILL
-      def indirect_ref_keyx(schema, id, doc_suffix, doc_type)
-        /^#{schema}_/.match?(id) and return id
-        ret = "#{schema}_#{id}"
-        doc_suffix && doc_type && doc_type != schema and
-          ret = "#{ret}_#{doc_suffix}"
-        ret
+                                       "#{schema}_#{id}_#{doc_suffix}"
+                                     else
+                                       "#{schema}_#{id}"
+                                     end
       end
 
       def update_indirect_refs_to_docs1(filec, key, file, bibitems, erefs)
@@ -196,13 +172,10 @@ module Metanorma
         parentid and file = "#{parentid}_#{file}"
         existing = a.text
         anchor = if url then existing
-                   else 
-                     #suffix_anchor_indirect(existing, suffix)
-                     #k = "#{existing}_#{file}"
-        #@ncnames[k] ||= Metanorma::Utils::to_ncname(k)
-        @indirect_keys[existing] ||= {}
-        @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
-                   end
+                 else
+                   @indirect_keys[existing] ||= {}
+                   @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
+                 end
         @updated_anchors[existing] or a.children = anchor
         @updated_anchors[anchor] = true
       end

data/lib/metanorma/collection/renderer/fileprocess.rb

@@ -11,15 +11,16 @@ module Metanorma
       # warn "metanorma compile -x html #{f.path}"
       def file_compile(file, filename, identifier)
         @files.get(identifier, :sectionsplit) and return
+        #require "debug"; binding.b
         opts = {
           format: :asciidoc,
-          extension_keys: @format,
+          extension_keys: @files.get(identifier, :format),
           output_dir: @outdir,
           type: @flavor,
         }.merge(compile_options_update(identifier))
         @compile.compile file, opts
         @files.set(identifier, :outputs, {})
-        file_compile_formats(filename, identifier)
+        file_compile_formats(filename, identifier, opts)
       end
 
       def compile_options_update(identifier)
@@ -34,19 +35,34 @@ module Metanorma
         ret
       end
 
-      def file_compile_formats(filename, identifier)
+=begin        
+      def allowed_extension_keys
+        ret = @format.dup
+        @directives.detect { |d| d.key == "individual-pdf" } or
+          ret.delete(:pdf)
+        @directives.detect { |d| d.key == "individual-doc" } or
+          ret.delete(:doc)
+        ret
+      end
+=end        
+
+      def file_compile_formats(filename, identifier, opts)
         f = @files.get(identifier, :outputs)
-        concatenate_presentation?({ format: @format }) and
-          @format << :presentation
-        @format.each do |e|
-          ext = @compile.processor.output_formats[e]
-          fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
-          (/html$/.match?(ext) && @files.get(identifier, :sectionsplit)) or
-            f[e] = File.join(@outdir, fn)
+        format = opts[:extension_keys]
+        concatenate_presentation?({ format: format }) and format << :presentation
+        format.each do |e|
+          file_compile_format(filename, identifier, e, f)
         end
         @files.set(identifier, :outputs, f)
       end
 
+      def file_compile_format(filename, identifier, format, output_formats)
+        ext = @compile.processor.output_formats[format]
+        fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
+        (/html$/.match?(ext) && @files.get(identifier, :sectionsplit)) or
+          output_formats[format] = File.join(@outdir, fn)
+      end
+
       def copy_file_to_dest(identifier)
         out = Pathname.new(@files.get(identifier, :out_path)).cleanpath
         out.absolute? and

data/lib/metanorma/collection/renderer/navigation.rb

@@ -55,7 +55,7 @@ module Metanorma
 
       def index?(mnf)
         mnf.index and return true
-        mnf.entry.detect { |e| index?(e) }
+        mnf.entry&.detect { |e| index?(e) }
       end
 
       def indexfile1(mnf)

data/lib/metanorma/collection/renderer/renderer.rb

@@ -15,7 +15,7 @@ module Metanorma
     class Renderer
       FORMATS = %i[html xml doc pdf].freeze
 
-      attr_accessor :isodoc, :nested
+      attr_accessor :isodoc, :isodoc_presxml, :nested
       attr_reader :xml, :compile, :compile_options, :documents, :outdir,
                   :manifest
 
@@ -45,6 +45,8 @@ module Metanorma
 
         # output processor for flavour
         @isodoc = Util::isodoc_create(@flavor, @lang, @script, @xml)
+        @isodoc_presxml = Util::isodoc_create(@flavor, @lang, @script, @xml,
+                                              presxml: true)
         @outdir = dir_name_cleanse(options[:output_folder])
         @coverpage = options[:coverpage] || collection.coverpage
         @format = ::Metanorma::Util.sort_extensions_execution(options[:format])
@@ -91,12 +93,22 @@ module Metanorma
         warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
         cr = new(col, File.dirname(col.file), options)
         cr.files
+        cr.rxl(options)
         cr.concatenate(col, options)
         options[:format]&.include?(:html) and cr.coverpage
         cr.flush_files
         cr
       end
 
+      def rxl(options)
+        @bibdata or return
+        options[:site_generate] and options[:format] << :xml
+        options[:format].include?(:rxl) or return
+        File.open(File.join(@outdir, "collection.rxl"), "w:UTF-8") do |f|
+          f.write(@bibdata.to_xml)
+        end
+      end
+
       def concatenate(col, options)
         warn "\n\n\n\n\nConcatenate: #{DateTime.now.strftime('%H:%M:%S')}"
         concatenate_presentation?(options) and
@@ -112,6 +124,7 @@ module Metanorma
       end
 
       def concatenate_prep(col, options)
+        warn options[:format]
         %i(xml presentation).each do |e|
           options[:format].include?(e) or next
           ext = e == :presentation ? "presentation.xml" : e.to_s

data/lib/metanorma/collection/renderer/utils.rb

@@ -12,14 +12,19 @@ module Metanorma
       end
 
       def dup_bibitem(docid, bib)
-        newbib = @files.get(docid, :bibdata).dup
+        newbib = deep_detached_clone(@files.get(docid, :bibdata))
         newbib.name = "bibitem"
         newbib["hidden"] = "true"
         newbib&.at("./*[local-name() = 'ext']")&.remove
         newbib["id"] = bib["id"]
+        bib["anchor"] and newbib["anchor"] = bib["anchor"]
         newbib
       end
 
+      def deep_detached_clone(node)
+        Nokogiri::XML(node.to_xml).root
+      end
+
       def get_bibitem_docid(bib, identifier)
         docid =
           bib.at(ns("./docidentifier[@type = 'metanorma-collection']")) ||
@@ -33,17 +38,10 @@ module Metanorma
         end
       end
 
-      def hide_refs(docxml)
-        docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or " \
-                        "@hidden = 'false'])]")).each do |f|
-          f["hidden"] = "true"
-        end
-      end
-
       def new_hidden_ref(xmldoc)
         ins = xmldoc.at(ns("bibliography")) or
           xmldoc.root << "<bibliography/>" and ins = xmldoc.at(ns("bibliography"))
-        ins.at(ns("./referenced[@hidden = 'true']")) or
+        ins.at(ns("./references[@hidden = 'true']")) or
           ins.add_child("<references hidden='true' normative='false'/>").first
       end
 
@@ -77,7 +75,7 @@ module Metanorma
         refs.each do |k, v|
           url = @files.url(v, {})
           ins << <<~XML
-            <bibitem id="#{k}">#{docid_xml(v)}<uri type='citation'>#{url}</uri></bibitem>
+            <bibitem id="#{k}" anchor="#{k}">#{docid_xml(v)}<uri type='citation'>#{url}</uri></bibitem>
           XML
         end
       end

data/lib/metanorma/collection/sectionsplit/collection.rb

@@ -39,6 +39,7 @@ module Metanorma
       end
 
       def collectionyaml(files, xml)
+        #warn xml.to_xml
         ret = {
           directives: ["presentation-xml", "bare-after-first"],
           bibdata: {