metanorma 1.3.5 → 1.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d4ccf8d6a09fd9345faa44ae7201b4f7ee8816a6f36245c26b2482abd3f7afa9
-  data.tar.gz: 7997e0bea6b752e07f7db96f19b6d136b7a9740c046a0850022b477a2223a85b
+  metadata.gz: 46ea9a7ce8316d94001c40043488f3472bcf639264e446bfb5bfaad88b6bed70
+  data.tar.gz: 7e4bfc4c4346c518d845052fd2dbb0b1f51f63e2951a317c18d938f9663cdea3
 SHA512:
-  metadata.gz: 162c6e5e79ff9f7f4f41b17444c68ec23168ee593173b72c251c8623f9a16916f6ae557bc581f39d220c2c6d68d6350c2e4f4de7c000a87e90f39bed7e825609
-  data.tar.gz: a92a8a6ddd41e404cb6a1e00e0a0aaa2ecd645723486769b8656bb09e10f9e595a9df81231713224f6f9781d17c1c6943135d978b5865d0c02842b5e0ccbb84c
+  metadata.gz: fdb777c1e35bbeeac65f3e5b0654ead9633a196340017da8cd4bd84342a9500a891a14ac19d83cc0cda0e5301728afdddf367fa3871eece62b389d93d0144238
+  data.tar.gz: c264222db4c7e2cbd7342aac8da9c5480a1d48f277ce2c5ef23000443d618379abb0769d111a8846733d60a53364706a65edd1a14de7ada03b94d16182c479bb

data/.github/workflows/rake.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
     steps:

data/Gemfile

@@ -6,6 +6,6 @@ git_source(:github) { |repo| "https://github.com/#{repo}" }
 
 gemspec
 
-if File.exist? 'Gemfile.devel'
-  eval File.read('Gemfile.devel'), nil, 'Gemfile.devel' # rubocop:disable Security/Eval
+if File.exist? "Gemfile.devel"
+  eval File.read("Gemfile.devel"), nil, "Gemfile.devel" # rubocop:disable Security/Eval
 end

data/lib/metanorma.rb

@@ -5,7 +5,6 @@ require "asciidoctor"
 require "metanorma/util"
 require "metanorma/config"
 require "metanorma/input"
-require "metanorma/output"
 require "metanorma/registry"
 require "metanorma/processor"
 require "metanorma/asciidoctor_extensions"

data/lib/metanorma/collection.rb

@@ -91,7 +91,9 @@ module Metanorma
       private
 
       def parse_xml(file)
-        xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
+        xml = Nokogiri::XML File.read(file, encoding: "UTF-8") do |config|
+          config.huge
+        end
         if (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata"))
           bd = Relaton::Cli.parse_xml b
         end

data/lib/metanorma/collection_fileparse.rb

@@ -11,17 +11,21 @@ module Metanorma
       xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n, {})
       xrefs.parse xml
       xrefs.get.each_with_object({}) do |(k, v), ret|
-        v[:type] ||= "clause"
-        ret[v[:type]] ||= {}
-        index = if v[:container] || v[:label].nil? || v[:label].empty?
-                  UUIDTools::UUID.random_create.to_s
-                else v[:label]
-                end
-        ret[v[:type]][index] = k
-        ret[v[:type]][v[:value]] = k if v[:value]
+        read_anchors1(k, v, ret)
       end
     end
 
+    def read_anchors1(key, val, ret)
+      val[:type] ||= "clause"
+      ret[val[:type]] ||= {}
+      index = if val[:container] || val[:label].nil? || val[:label].empty?
+                UUIDTools::UUID.random_create.to_s
+              else val[:label]
+              end
+      ret[val[:type]][index] = key
+      ret[val[:type]][val[:value]] = key if val[:value]
+    end
+
     # @param id [String]
     # @param read [Boolean]
     # @return [Array<String, nil>]
@@ -33,30 +37,36 @@ module Metanorma
 
     # @param bib [Nokogiri::XML::Element]
     # @param identifier [String]
-    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize
       docid = bib&.at(ns("./docidentifier"))&.children&.to_xml
-      unless @files[docid]
-        error = "[metanorma] Cannot find crossreference to document #{docid} "\
-          "in document #{identifier}."
-        @log.add("Cross-References", nil, error)
-        Util.log(error, :warning)
-        return
-      end
-      id = bib["id"]
-      newbib = @files[docid][:bibdata].dup
-      newbib.name = "bibitem"
-      newbib["id"] = id
-      newbib["hidden"] = "true"
-      newbib&.at("./*[local-name() = 'ext']")&.remove
+      return fail_update_bibitem(docid, identifier) unless @files[docid]
+
+      newbib = dup_bibitem(docid, bib)
       bib.replace(newbib)
       _file, url = targetfile(@files[docid], relative: true, read: false,
-                                             doc: !@files[docid][:attachment])
+                              doc: !@files[docid][:attachment])
       uri_node = Nokogiri::XML::Node.new "uri", newbib
       uri_node[:type] = "citation"
       uri_node.content = url
       newbib.at(ns("./docidentifier")).previous = uri_node
     end
 
+    def fail_update_bibitem(docid, identifier)
+      error = "[metanorma] Cannot find crossreference to document #{docid} "\
+        "in document #{identifier}."
+      @log.add("Cross-References", nil, error)
+      Util.log(error, :warning)
+    end
+
+    def dup_bibitem(docid, bib)
+      newbib = @files[docid][:bibdata].dup
+      newbib.name = "bibitem"
+      newbib["hidden"] = "true"
+      newbib&.at("./*[local-name() = 'ext']")&.remove
+      newbib["id"] = bib["id"]
+      newbib
+    end
+
     # Resolves direct links to other files in collection
     # (repo(current-metanorma-collection/x),
     # and indirect links to other files in collection
@@ -67,7 +77,7 @@ module Metanorma
     # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to anchor to filename
     # @return [String] XML content
     def update_xrefs(file, identifier, internal_refs)
-      docxml = Nokogiri::XML(file)
+      docxml = Nokogiri::XML(file) { |config| config.huge }
       update_indirect_refs_to_docs(docxml, internal_refs)
       add_document_suffix(identifier, docxml)
       update_direct_refs_to_docs(docxml, identifier)
@@ -106,12 +116,18 @@ module Metanorma
     # Preferably with anchor, and is a job to realise dynamic lookup
     # of localities.
     def update_direct_refs_to_docs(docxml, identifier)
+      erefs = docxml.xpath(ns("//eref"))
+        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
+        m[:citeas][i["citeas"]] = true
+        m[:bibitemid][i["bibitemid"]] = true
+      end
       docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
         docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
         next unless docid && %r{^current-metanorma-collection/}.match(docid)
 
         update_bibitem(b, identifier)
-        update_anchors(b, docxml, docid)
+        docid = b&.at(ns("./docidentifier"))&.children&.to_xml or next
+        erefs[:citeas][docid] and update_anchors(b, docxml, docid)
       end
     end
 
@@ -128,6 +144,9 @@ module Metanorma
     def update_indirect_refs_to_docs1(docxml, schema, id, file)
       docxml.xpath(ns("//eref[@bibitemid = '#{schema}_#{id}']")).each do |e|
         e["citeas"] = file
+        if a = e.at(ns(".//locality[@type = 'anchor']/referenceFrom"))
+          a.children = "#{a.text}_#{Metanorma::Utils::to_ncname(file)}"
+        end
       end
       docid = docxml.at(ns("//bibitem[@id = '#{schema}_#{id}']/"\
                            "docidentifier[@type = 'repository']")) or return
@@ -137,21 +156,19 @@ module Metanorma
 
     # update crossrefences to other documents, to include
     # disambiguating document suffix on id
-    def update_anchors(bib, docxml, _id) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      docid = bib&.at(ns("./docidentifier"))&.children.to_xml
+    def update_anchors(bib, docxml, docid) # rubocop:disable Metrics/AbcSize
       docxml.xpath("//xmlns:eref[@citeas = '#{docid}']").each do |e|
-        if @files[docid]
-          update_anchor_loc(bib, e, docid)
+        if @files[docid] then update_anchor_loc(bib, e, docid)
         else
-          e << "<strong>** Unresolved reference to document #{docid}, "\
-            "id #{e['bibitemid']}</strong>"
+          e << "<strong>** Unresolved reference to document #{docid} "\
+            "from eref</strong>"
         end
       end
     end
 
-    def update_anchor_loc(bib, e, docid)
-      loc = e.at(ns(".//locality[@type = 'anchor']")) or
-        return update_anchor_create_loc(bib, e, docid)
+    def update_anchor_loc(bib, eref, docid)
+      loc = eref.at(ns(".//locality[@type = 'anchor']")) or
+        return update_anchor_create_loc(bib, eref, docid)
       document_suffix = Metanorma::Utils::to_ncname(docid)
       ref = loc.at(ns("./referenceFrom")) || return
       anchor = "#{ref.text}_#{document_suffix}"
@@ -164,18 +181,14 @@ module Metanorma
 
     # if there is a crossref to another document, with no anchor, retrieve the
     # anchor given the locality, and insert it into the crossref
-    def update_anchor_create_loc(bib, eref, docid)
+    def update_anchor_create_loc(_bib, eref, docid)
       ins = eref.at(ns("./localityStack")) or return
       type = ins&.at(ns("./locality/@type"))&.text
       type = "clause" if type == "annex"
       ref = ins&.at(ns("./locality/referenceFrom"))&.text
       anchor = @files[docid][:anchors].dig(type, ref) or return
-      ref_from = Nokogiri::XML::Node.new "referenceFrom", bib
-      ref_from.content = anchor.sub(/^_/, "")
-      locality = Nokogiri::XML::Node.new "locality", bib
-      locality[:type] = "anchor"
-      locality.add_child ref_from
-      ins << locality
+      ins << "<locality type='anchor'><referenceFrom>#{anchor.sub(/^_/, '')}"\
+        "</referenceFrom></locality>"
     end
 
     # gather internal bibitem references
@@ -186,7 +199,7 @@ module Metanorma
         file, = targetfile(x, read: true)
         Nokogiri::XML(file)
           .xpath(ns("//bibitem[@type = 'internal']/"\
-        "docidentifier[@type = 'repository']")).each do |d|
+                    "docidentifier[@type = 'repository']")).each do |d|
           a = d.text.split(%r{/}, 2)
           a.size > 1 or next
           refs[a[0]] ||= {}
@@ -211,10 +224,11 @@ module Metanorma
 
     def locate_internal_refs1(refs, identifier, filedesc)
       file, _filename = targetfile(filedesc, read: true)
-      docxml = Nokogiri::XML(file)
+      xml = Nokogiri::XML(file) { |config| config.huge }
+      t = xml.xpath("//*/@id").each_with_object({}) { |i, x| x[i.text] = true }
       refs.each do |schema, ids|
-        ids.each_key do |id|
-          n = docxml.at("//*[@id = '#{id}']") and
+        ids.keys.select { |id| t[id] }.each do |id|
+          n = xml.at("//*[@id = '#{id}']") and
             n.at("./ancestor-or-self::*[@type = '#{schema}']") and
             refs[schema][id] = identifier
         end

data/lib/metanorma/collection_fileprocess.rb

@@ -27,8 +27,69 @@ module Metanorma
           files[identifier][:anchors] = read_anchors(xml)
           files[identifier][:bibdata] = xml.at(ns("//bibdata"))
         end
+        files[identifier][:bibitem] = files[identifier][:bibdata].dup
+        files[identifier][:bibitem].name = "bibitem"
+        files[identifier][:bibitem]["hidden"] = "true"
+        files[identifier][:bibitem]&.at("./*[local-name() = 'ext']")&.remove
       end
-      files
+      add_section_split(files)
+    end
+
+    def add_section_split(files)
+      files.keys.each_with_object({}) do |k, m|
+        if files[k][:sectionsplit] == "true" && !files[k]["attachment"]
+          s, manifest = sectionsplit(files[k][:ref])
+          s.each_with_index do |f1, i|
+            add_section_split_instance(f1, m, k, i, files)
+          end
+          m["#{k}:index.html"] = add_section_split_cover(files, manifest, k)
+        end
+        m[k] = files[k]
+      end
+    end
+
+    def add_section_split_cover(files, manifest, ident)
+      cover = section_split_cover(manifest, ident)
+      files[ident][:out_path] = cover
+      { attachment: true, index: false, out_path: cover,
+        ref: File.join(File.dirname(manifest.file), cover) }
+    end
+
+    def section_split_cover(col, ident)
+      dir = File.dirname(col.file)
+      @compile.collection_setup(nil, dir)
+      CollectionRenderer.new(col, dir,
+                             output_folder: "#{ident}_collection",
+                             format: %i(html),
+                             coverpage: File.join(dir, "cover.html")).coverpage
+      FileUtils.mv "#{ident}_collection/index.html",
+                   File.join(dir, "#{ident}_index.html")
+      FileUtils.rm_rf "#{ident}_collection"
+      "#{ident}_index.html"
+    end
+
+    def add_section_split_instance(file, manifest, key, idx, files)
+      dir = File.dirname(files[key][:ref])
+      presfile = File.join(dir, File.basename(file[:url]))
+      manifest["#{key} #{file[:title]}"] =
+        { parentid: key, presentationxml: true, type: "fileref",
+          rel_path: file[:url], out_path: File.basename(file[:url]),
+          anchors: read_anchors(Nokogiri::XML(File.read(presfile))),
+          bibdata: files[key][:bibdata], ref: presfile }
+      manifest["#{key} #{file[:title]}"][:bare] = true unless idx.zero?
+    end
+
+    def sectionsplit(file)
+      @compile.compile(
+        file, { format: :asciidoc, extension_keys: [:presentation] }
+        .merge(@compile_options)
+      )
+      r = file.sub(/\.xml$/, ".presentation.xml")
+      xml = Nokogiri::XML(File.read(r))
+      s = @compile.sectionsplit(xml, File.basename(r), File.dirname(r))
+        .sort_by { |f| f[:order] }
+      [s, @compile.collection_manifest(File.basename(r), s, xml, nil,
+                                       File.dirname(r))]
     end
 
     # rel_path is the source file address, determined relative to the YAML.
@@ -43,7 +104,11 @@ module Metanorma
             else
               { type: "id", ref: ref["id"] }
             end
-      ret[:attachment] = ref["attachment"] if ref["attachment"]
+      %i(attachment sectionsplit index).each do |s|
+        ret[s] = ref[s.to_s] if ref[s.to_s]
+      end
+      ret[:presentationxml] = ref["presentation-xml"] if ref["presentation-xml"]
+      ret[:bareafterfirst] = ref["bare-after-first"] if ref["bare-after-first"]
       ret
     end
 
@@ -95,22 +160,51 @@ module Metanorma
     end
 
     # compile and output individual file in collection
+    # warn "metanorma compile -x html #{f.path}"
     def file_compile(file, filename, identifier)
-      # warn "metanorma compile -x html #{f.path}"
-      Array(@directives).include?("presentation-xml") and
-        @compile_options.merge!(passthrough_presentation_xml: true)
-      c = Compile.new
-      c.compile file.path, { format: :asciidoc,
-                             extension_keys: @format }.merge(@compile_options)
+      return if @files[identifier][:sectionsplit] == "true"
+
+      @compile.compile file.path, { format: :asciidoc, extension_keys: @format }
+        .merge(compile_options(identifier))
       @files[identifier][:outputs] = {}
+      file_compile_formats(file, filename, identifier)
+    end
+
+    def compile_options(identifier)
+      ret = @compile_options.dup
+      Array(@directives).include?("presentation-xml") ||
+        @files[identifier][:presentationxml] and
+        ret.merge!(passthrough_presentation_xml: true)
+      @files[identifier][:sectionsplit] == "true" and
+        ret.merge!(sectionsplit: "true")
+      @files[identifier][:bare] == true and
+        ret.merge!(bare: true)
+      ret
+    end
+
+    def file_compile_formats(file, filename, identifier)
       @format.each do |e|
-        ext = c.processor.output_formats[e]
-        fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
-        FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
-        @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        ext = @compile.processor.output_formats[e]
+        fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
+        if /html$/.match?(ext) && @files[identifier][:sectionsplit]
+          # file_sectionsplit_copy(file, fn, identifier, ext, e)
+        else
+          FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"),
+                       File.join(@outdir, fn)
+          @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        end
       end
     end
 
+    def file_sectionsplit_copy(file, base, identifier, ext, format)
+      dir = file.path.sub(/\.xml$/, ".#{ext}_collection")
+      files = Dir.glob("#{dir}/*.#{ext}")
+      FileUtils.cp files, @outdir
+      cover = File.join(@outdir, base.sub(/\.html$/, ".index.html"))
+      FileUtils.cp File.join(dir, "index.html"), cover
+      @files[identifier][:outputs][format] = cover
+    end
+
     def copy_file_to_dest(fileref)
       dest = File.join(@outdir, fileref[:out_path])
       FileUtils.mkdir_p(File.dirname(dest))
@@ -120,6 +214,7 @@ module Metanorma
     # process each file in the collection
     # files are held in memory, and altered as postprocessing
     def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      warn "\n\n\n\n\nInternal Refs: #{DateTime.now.strftime('%H:%M:%S')}"
       internal_refs = locate_internal_refs
       @files.each_with_index do |(identifier, x), i|
         i.positive? && Array(@directives).include?("bare-after-first") and
@@ -127,6 +222,7 @@ module Metanorma
         if x[:attachment] then copy_file_to_dest(x)
         else
           file, filename = targetfile(x, read: true)
+          warn "\n\n\n\n\nProcess #{filename}: #{DateTime.now.strftime('%H:%M:%S')}"
           file = update_xrefs(file, identifier, internal_refs)
           Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
             f.write(file)

data/lib/metanorma/collection_manifest.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "util"
 
 module Metanorma
@@ -46,8 +47,10 @@ module Metanorma
       def parse_docref(mnf)
         mnf.xpath("xmlns:docref").map do |dr|
           h = { "identifier" => dr.at("identifier").children.to_xml }
-          dr[:fileref] and h["fileref"] = dr[:fileref]
-          h["attachment"] = dr[:attachment] if dr[:attachment]
+          %i(fileref attachment sectionsplit index).each do |s|
+            h[s.to_s] = dr[s] if dr[s]
+          end
+          h["presentation-xml"] = dr[:presentationxml] if dr[:presentationxml]
           h
         end
       end
@@ -59,7 +62,7 @@ module Metanorma
       @manifest.each { |mnf| mnf.collection = col }
     end
 
-    # @param dir [String] path to coolection
+    # @param dir [String] path to collection
     # @return [Hash<String, Metanorma::Document>]
     def documents(dir = "")
       docs = @docref.each_with_object({}) do |dr, m|
@@ -67,7 +70,7 @@ module Metanorma
 
         m[dr["identifier"]] = Document.parse_file(
           File.join(dir, dr["fileref"]),
-          dr["attachment"], dr["identifier"]
+          dr["attachment"], dr["identifier"], dr["index"]
         )
         m
       end
@@ -115,7 +118,16 @@ module Metanorma
 
     def docref_to_xml_attrs(elem, docref)
       elem[:fileref] = @disambig.source2dest_filename(docref["fileref"])
-      elem[:attachment] = docref["attachment"] if docref["attachment"]
+      %i(attachment sectionsplit).each do |i|
+        elem[i] = docref[i.to_s] if docref[i.to_s]
+      end
+      elem[:index] = docref.has_key?("index") ? docref["index"] : "true"
+      elem[:presentationxml] = "true" if docref["presentation-xml"] &&
+        [true, "true"].include?(docref["presentation-xml"])
+      docref_to_xml_attrs_id(elem, docref)
+    end
+
+    def docref_to_xml_attrs_id(elem, docref)
       if collection.directives.include?("documents-inline")
         id = collection.documents.find_index do |k, _|
           k == docref["identifier"]