metanorma 1.3.4 → 1.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c92210eec85835a4e8be4a85a1c1dbe84cc1b88163e0f3c9c9be136f0019cb6e
-  data.tar.gz: 066d89b3df0a8865f52e3e70b257a1a4ce3c71eb9f8b6840a87b9046743536d1
+  metadata.gz: d8b82f79d4ce3a18564bd84c137faeca4daf763e2284464f390476fb881a0996
+  data.tar.gz: f6a33995af907243fd0ae9ebff3676467455f21cd502089da6599783b4ba4e86
 SHA512:
-  metadata.gz: d23d488ab60995b4745184cc5a22aaaf1c9b24694186b8cfb0d7359ecaea1a999fef99dcc71c15944d5e9fbdd377f0739b35f79528f46071a7a701b512fd3108
-  data.tar.gz: 70dd774b5b69a08661cacf0d2601b0a92cb45624848f23f6f3a397df02ef9c496ff44a64f1c1fc083d07bbfa8754921dafe6168d0472ab776cb1453b064313ce
+  metadata.gz: 484b71a094f6e23f5e512ffcbc8eac5bed0cd6d5b3fc6e952a28ce05e0e9eef3b64bff995692fd43f36b0c7888814b20318559aeeb0d9dee7900baf0847d95b8
+  data.tar.gz: 508757811e5a63b30b91d4353fdc2a983bfe269da85f39b9be50d0c6a904113965ae53c40052cd43fb38ee740c350b9922d114bfd21833383a525782d278f95c

data/.github/workflows/rake.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
     steps:

data/.rubocop.yml

@@ -7,4 +7,4 @@ inherit_from:
 # ...
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5

data/Gemfile

@@ -6,6 +6,6 @@ git_source(:github) { |repo| "https://github.com/#{repo}" }
 
 gemspec
 
-if File.exist? 'Gemfile.devel'
-  eval File.read('Gemfile.devel'), nil, 'Gemfile.devel' # rubocop:disable Security/Eval
+if File.exist? "Gemfile.devel"
+  eval File.read("Gemfile.devel"), nil, "Gemfile.devel" # rubocop:disable Security/Eval
 end

data/lib/metanorma.rb

@@ -5,7 +5,6 @@ require "asciidoctor"
 require "metanorma/util"
 require "metanorma/config"
 require "metanorma/input"
-require "metanorma/output"
 require "metanorma/registry"
 require "metanorma/processor"
 require "metanorma/asciidoctor_extensions"

data/lib/metanorma/collection.rb

@@ -4,6 +4,7 @@ require "relaton"
 require "relaton/cli"
 require "metanorma/collection_manifest"
 require "metanorma-utils"
+require_relative "util"
 
 module Metanorma
   # Metanorma collection of documents
@@ -18,6 +19,8 @@ module Metanorma
     # @return [Hash<String, Metanorma::Document>]
     attr_accessor :documents
 
+    attr_accessor :disambig
+
     # @param file [String] path to source file
     # @param directives [Array<String>] documents-inline to inject the XML into
     #   the collection manifest; documents-external to keeps them outside
@@ -41,6 +44,7 @@ module Metanorma
       @prefatory = args[:prefatory]
       @final = args[:final]
       @log = Metanorma::Utils::Log.new
+      @disambig = Util::DisambigFiles.new
     end
 
     # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
@@ -87,7 +91,9 @@ module Metanorma
       private
 
       def parse_xml(file)
-        xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
+        xml = Nokogiri::XML File.read(file, encoding: "UTF-8") do |config|
+          config.huge
+        end
         if (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata"))
           bd = Relaton::Cli.parse_xml b
         end

data/lib/metanorma/collection_fileparse.rb

@@ -11,15 +11,21 @@ module Metanorma
       xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n, {})
       xrefs.parse xml
       xrefs.get.each_with_object({}) do |(k, v), ret|
-        ret[v[:type]] ||= {}
-        index = if v[:container] || v[:label].nil? || v[:label].empty?
-                  UUIDTools::UUID.random_create.to_s
-                else v[:label]
-                end
-        ret[v[:type]][index] = k
+        read_anchors1(k, v, ret)
       end
     end
 
+    def read_anchors1(key, val, ret)
+      val[:type] ||= "clause"
+      ret[val[:type]] ||= {}
+      index = if val[:container] || val[:label].nil? || val[:label].empty?
+                UUIDTools::UUID.random_create.to_s
+              else val[:label]
+              end
+      ret[val[:type]][index] = key
+      ret[val[:type]][val[:value]] = key if val[:value]
+    end
+
     # @param id [String]
     # @param read [Boolean]
     # @return [Array<String, nil>]
@@ -31,29 +37,36 @@ module Metanorma
 
     # @param bib [Nokogiri::XML::Element]
     # @param identifier [String]
-    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      docid = bib&.at(ns("./docidentifier"))&.text
-      unless @files[docid]
-        error = "[metanorma] Cannot find crossreference to document #{docid} "\
-          "in document #{identifier}."
-        @log.add("Cross-References", nil, error)
-        Util.log(error, :warning)
-        return
-      end
-      id = bib["id"]
-      newbib = bib.replace(@files[docid][:bibdata])
-      newbib.name = "bibitem"
-      newbib["id"] = id
-      newbib["hidden"] = "true"
-      newbib&.at(ns("./ext"))&.remove
+    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize
+      docid = bib&.at(ns("./docidentifier"))&.children&.to_xml
+      return fail_update_bibitem(docid, identifier) unless @files[docid]
+
+      newbib = dup_bibitem(docid, bib)
+      bib.replace(newbib)
       _file, url = targetfile(@files[docid], relative: true, read: false,
-                                             doc: !@files[docid][:attachment])
+                              doc: !@files[docid][:attachment])
       uri_node = Nokogiri::XML::Node.new "uri", newbib
       uri_node[:type] = "citation"
       uri_node.content = url
       newbib.at(ns("./docidentifier")).previous = uri_node
     end
 
+    def fail_update_bibitem(docid, identifier)
+      error = "[metanorma] Cannot find crossreference to document #{docid} "\
+        "in document #{identifier}."
+      @log.add("Cross-References", nil, error)
+      Util.log(error, :warning)
+    end
+
+    def dup_bibitem(docid, bib)
+      newbib = @files[docid][:bibdata].dup
+      newbib.name = "bibitem"
+      newbib["hidden"] = "true"
+      newbib&.at("./*[local-name() = 'ext']")&.remove
+      newbib["id"] = bib["id"]
+      newbib
+    end
+
     # Resolves direct links to other files in collection
     # (repo(current-metanorma-collection/x),
     # and indirect links to other files in collection
@@ -64,12 +77,12 @@ module Metanorma
     # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to anchor to filename
     # @return [String] XML content
     def update_xrefs(file, identifier, internal_refs)
-      docxml = Nokogiri::XML(file)
+      docxml = Nokogiri::XML(file) { |config| config.huge }
       update_indirect_refs_to_docs(docxml, internal_refs)
       add_document_suffix(identifier, docxml)
       update_direct_refs_to_docs(docxml, identifier)
       svgmap_resolve(datauri_encode(docxml))
-      docxml.xpath(ns("//references[not(./bibitem[not(@hidden) or "\
+      docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or "\
                       "@hidden = 'false'])]")).each do |f|
         f["hidden"] = "true"
       end
@@ -103,12 +116,18 @@ module Metanorma
     # Preferably with anchor, and is a job to realise dynamic lookup
     # of localities.
     def update_direct_refs_to_docs(docxml, identifier)
+      erefs = docxml.xpath(ns("//eref"))
+        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
+        m[:citeas][i["citeas"]] = true
+        m[:bibitemid][i["bibitemid"]] = true
+      end
       docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
         docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
         next unless docid && %r{^current-metanorma-collection/}.match(docid)
 
         update_bibitem(b, identifier)
-        update_anchors(b, docxml, docid)
+        docid = b&.at(ns("./docidentifier"))&.children&.to_xml or next
+        erefs[:citeas][docid] and update_anchors(b, docxml, docid)
       end
     end
 
@@ -125,6 +144,9 @@ module Metanorma
     def update_indirect_refs_to_docs1(docxml, schema, id, file)
       docxml.xpath(ns("//eref[@bibitemid = '#{schema}_#{id}']")).each do |e|
         e["citeas"] = file
+        if a = e.at(ns(".//locality[@type = 'anchor']/referenceFrom"))
+          a.children = "#{a.text}_#{Metanorma::Utils::to_ncname(file)}"
+        end
       end
       docid = docxml.at(ns("//bibitem[@id = '#{schema}_#{id}']/"\
                            "docidentifier[@type = 'repository']")) or return
@@ -134,21 +156,19 @@ module Metanorma
 
     # update crossrefences to other documents, to include
     # disambiguating document suffix on id
-    def update_anchors(bib, docxml, _id) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      docid = bib&.at(ns("./docidentifier"))&.text
+    def update_anchors(bib, docxml, docid) # rubocop:disable Metrics/AbcSize
       docxml.xpath("//xmlns:eref[@citeas = '#{docid}']").each do |e|
-        if @files[docid]
-          update_anchor_loc(bib, e, docid)
+        if @files[docid] then update_anchor_loc(bib, e, docid)
         else
-          e << "<strong>** Unresolved reference to document #{docid}, "\
-            "id #{e['bibitemid']}</strong>"
+          e << "<strong>** Unresolved reference to document #{docid} "\
+            "from eref</strong>"
         end
       end
     end
 
-    def update_anchor_loc(bib, e, docid)
-      loc = e.at(ns(".//locality[@type = 'anchor']")) or
-        return update_anchor_create_loc(bib, e, docid)
+    def update_anchor_loc(bib, eref, docid)
+      loc = eref.at(ns(".//locality[@type = 'anchor']")) or
+        return update_anchor_create_loc(bib, eref, docid)
       document_suffix = Metanorma::Utils::to_ncname(docid)
       ref = loc.at(ns("./referenceFrom")) || return
       anchor = "#{ref.text}_#{document_suffix}"
@@ -161,17 +181,14 @@ module Metanorma
 
     # if there is a crossref to another document, with no anchor, retrieve the
     # anchor given the locality, and insert it into the crossref
-    def update_anchor_create_loc(bib, e, docid)
-      ins = e.at(ns("./localityStack")) || return
+    def update_anchor_create_loc(_bib, eref, docid)
+      ins = eref.at(ns("./localityStack")) or return
       type = ins&.at(ns("./locality/@type"))&.text
+      type = "clause" if type == "annex"
       ref = ins&.at(ns("./locality/referenceFrom"))&.text
-      (anchor = @files[docid][:anchors][type][ref]) || return
-      ref_from = Nokogiri::XML::Node.new "referenceFrom", bib
-      ref_from.content = anchor.sub(/^_/, "")
-      locality = Nokogiri::XML::Node.new "locality", bib
-      locality[:type] = "anchor"
-      locality.add_child ref_from
-      ins << locality
+      anchor = @files[docid][:anchors].dig(type, ref) or return
+      ins << "<locality type='anchor'><referenceFrom>#{anchor.sub(/^_/, '')}"\
+        "</referenceFrom></locality>"
     end
 
     # gather internal bibitem references
@@ -182,7 +199,7 @@ module Metanorma
         file, = targetfile(x, read: true)
         Nokogiri::XML(file)
           .xpath(ns("//bibitem[@type = 'internal']/"\
-        "docidentifier[@type = 'repository']")).each do |d|
+                    "docidentifier[@type = 'repository']")).each do |d|
           a = d.text.split(%r{/}, 2)
           a.size > 1 or next
           refs[a[0]] ||= {}
@@ -207,10 +224,11 @@ module Metanorma
 
     def locate_internal_refs1(refs, identifier, filedesc)
       file, _filename = targetfile(filedesc, read: true)
-      docxml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
+      t = xml.xpath("//*/@id").each_with_object({}) { |i, x| x[i.text] = true }
       refs.each do |schema, ids|
-        ids.each_key do |id|
-          n = docxml.at("//*[@id = '#{id}']") and
+        ids.keys.select { |id| t[id] }.each do |id|
+          n = xml.at("//*[@id = '#{id}']") and
             n.at("./ancestor-or-self::*[@type = '#{schema}']") and
             refs[schema][id] = identifier
         end

data/lib/metanorma/collection_fileprocess.rb

@@ -15,7 +15,7 @@ module Metanorma
     def read_files(path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       files = {}
       @xml.xpath(ns("//docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
+        identifier = d.at(ns("./identifier")).children.to_xml
         files[identifier] = file_entry(d, identifier, path)
         if files[identifier][:attachment]
           files[identifier][:bibdata] = Metanorma::Document
@@ -27,22 +27,88 @@ module Metanorma
           files[identifier][:anchors] = read_anchors(xml)
           files[identifier][:bibdata] = xml.at(ns("//bibdata"))
         end
+        files[identifier][:bibitem] = files[identifier][:bibdata].dup
+        files[identifier][:bibitem].name = "bibitem"
+        files[identifier][:bibitem]["hidden"] = "true"
+        files[identifier][:bibitem]&.at("./*[local-name() = 'ext']")&.remove
       end
-      files
+      add_section_split(files)
+    end
+
+    def add_section_split(files)
+      files.keys.each_with_object({}) do |k, m|
+        if files[k][:sectionsplit] == "true" && !files[k]["attachment"]
+          s, manifest = sectionsplit(files[k][:ref])
+          s.each_with_index do |f1, i|
+            add_section_split_instance(f1, m, k, i, files)
+          end
+          m["#{k}:index.html"] = add_section_split_cover(files, manifest, k)
+        end
+        m[k] = files[k]
+      end
+    end
+
+    def add_section_split_cover(files, manifest, ident)
+      cover = section_split_cover(manifest, ident)
+      files[ident][:out_path] = cover
+      { attachment: true, index: false, out_path: cover,
+        ref: File.join(File.dirname(manifest.file), cover) }
+    end
+
+    def section_split_cover(col, ident)
+      dir = File.dirname(col.file)
+      @compile.collection_setup(nil, dir)
+      CollectionRenderer.new(col, dir,
+                             output_folder: "#{ident}_collection",
+                             format: %i(html),
+                             coverpage: File.join(dir, "cover.html")).coverpage
+      FileUtils.mv "#{ident}_collection/index.html",
+                   File.join(dir, "#{ident}_index.html")
+      FileUtils.rm_rf "#{ident}_collection"
+      "#{ident}_index.html"
+    end
+
+    def add_section_split_instance(file, manifest, key, idx, files)
+      dir = File.dirname(files[key][:ref])
+      presfile = File.join(dir, File.basename(file[:url]))
+      manifest["#{key} #{file[:title]}"] =
+        { parentid: key, presentationxml: true, type: "fileref",
+          rel_path: file[:url], out_path: File.basename(file[:url]),
+          anchors: read_anchors(Nokogiri::XML(File.read(presfile))),
+          bibdata: files[key][:bibdata], ref: presfile }
+      manifest["#{key} #{file[:title]}"][:bare] = true unless idx.zero?
+    end
+
+    def sectionsplit(file)
+      @compile.compile(
+        file, { format: :asciidoc, extension_keys: [:presentation] }
+        .merge(@compile_options)
+      )
+      r = file.sub(/\.xml$/, ".presentation.xml")
+      xml = Nokogiri::XML(File.read(r))
+      s = @compile.sectionsplit(xml, File.basename(r), File.dirname(r))
+        .sort_by { |f| f[:order] }
+      [s, @compile.collection_manifest(File.basename(r), s, xml, nil,
+                                       File.dirname(r))]
     end
 
     # rel_path is the source file address, determined relative to the YAML.
     # out_path is the destination file address, with any references outside
     # the working directory (../../...) truncated
-    def file_entry(docref, identifier, _path)
-      ret = if docref["fileref"]
+    def file_entry(ref, identifier, _path)
+      out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
+      ret = if ref["fileref"]
               { type: "fileref", ref: @documents[identifier].file,
-                rel_path: docref["fileref"],
-                out_path: Util::source2dest_filename(docref["fileref"]) }
+                rel_path: ref["fileref"],
+                out_path: out }
             else
-              { type: "id", ref: docref["id"] }
+              { type: "id", ref: ref["id"] }
             end
-      ret[:attachment] = docref["attachment"] if docref["attachment"]
+      %i(attachment sectionsplit index).each do |s|
+        ret[s] = ref[s.to_s] if ref[s.to_s]
+      end
+      ret[:presentationxml] = ref["presentation-xml"] if ref["presentation-xml"]
+      ret[:bareafterfirst] = ref["bare-after-first"] if ref["bare-after-first"]
       ret
     end
 
@@ -74,9 +140,9 @@ module Metanorma
     # @return [Array<String, nil>]
     def targetfile(data, options)
       options = { read: false, doc: true, relative: false }.merge(options)
-      path = options[:relative] ? data[:out_path] : data[:ref]
+      path = options[:relative] ? data[:rel_path] : data[:ref]
       if data[:type] == "fileref"
-        ref_file path, options[:read], options[:doc]
+        ref_file path, data[:out_path], options[:read], options[:doc]
       else
         xml_file data[:id], options[:read]
       end
@@ -86,43 +152,77 @@ module Metanorma
     # @param read [Boolean]
     # @param doc [Boolean]
     # @return [Array<String, nil>]
-    def ref_file(ref, read, doc)
+    def ref_file(ref, out, read, doc)
       file = File.read(ref, encoding: "utf-8") if read
-      filename = ref.dup
+      filename = out.dup
       filename.sub!(/\.xml$/, ".html") if doc
       [file, filename]
     end
 
     # compile and output individual file in collection
+    # warn "metanorma compile -x html #{f.path}"
     def file_compile(file, filename, identifier)
-      # warn "metanorma compile -x html #{f.path}"
-      c = Compile.new
-      c.compile file.path, { format: :asciidoc,
-                             extension_keys: @format }.merge(@compile_options)
+      return if @files[identifier][:sectionsplit] == "true"
+
+      @compile.compile file.path, { format: :asciidoc, extension_keys: @format }
+        .merge(compile_options(identifier))
       @files[identifier][:outputs] = {}
+      file_compile_formats(file, filename, identifier)
+    end
+
+    def compile_options(identifier)
+      ret = @compile_options.dup
+      Array(@directives).include?("presentation-xml") ||
+        @files[identifier][:presentationxml] and
+        ret.merge!(passthrough_presentation_xml: true)
+      @files[identifier][:sectionsplit] == "true" and
+        ret.merge!(sectionsplit: "true")
+      @files[identifier][:bare] == true and
+        ret.merge!(bare: true)
+      ret
+    end
+
+    def file_compile_formats(file, filename, identifier)
       @format.each do |e|
-        ext = c.processor.output_formats[e]
-        fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
-        FileUtils.mv file.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
-        @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        ext = @compile.processor.output_formats[e]
+        fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
+        if /html$/.match?(ext) && @files[identifier][:sectionsplit]
+          # file_sectionsplit_copy(file, fn, identifier, ext, e)
+        else
+          FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"),
+                       File.join(@outdir, fn)
+          @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        end
       end
     end
 
+    def file_sectionsplit_copy(file, base, identifier, ext, format)
+      dir = file.path.sub(/\.xml$/, ".#{ext}_collection")
+      files = Dir.glob("#{dir}/*.#{ext}")
+      FileUtils.cp files, @outdir
+      cover = File.join(@outdir, base.sub(/\.html$/, ".index.html"))
+      FileUtils.cp File.join(dir, "index.html"), cover
+      @files[identifier][:outputs][format] = cover
+    end
+
     def copy_file_to_dest(fileref)
-      _file, filename = targetfile(fileref, read: true, doc: false)
       dest = File.join(@outdir, fileref[:out_path])
       FileUtils.mkdir_p(File.dirname(dest))
-      FileUtils.cp filename, dest
+      FileUtils.cp fileref[:ref], dest
     end
 
     # process each file in the collection
     # files are held in memory, and altered as postprocessing
     def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      warn "\n\n\n\n\nInternal Refs: #{DateTime.now.strftime('%H:%M:%S')}"
       internal_refs = locate_internal_refs
-      @files.each do |identifier, x|
+      @files.each_with_index do |(identifier, x), i|
+        i.positive? && Array(@directives).include?("bare-after-first") and
+          @compile_options.merge!(bare: true)
         if x[:attachment] then copy_file_to_dest(x)
         else
           file, filename = targetfile(x, read: true)
+          warn "\n\n\n\n\nProcess #{filename}: #{DateTime.now.strftime('%H:%M:%S')}"
           file = update_xrefs(file, identifier, internal_refs)
           Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
             f.write(file)