metanorma 2.0.6 → 2.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f27c1f604a729b827832e208b2ca5db54b8db3f5b3f69fe905e0f0ae87a08f53
-  data.tar.gz: 781c8396a4c325a34c456ba935e40aba94d4062b283b475cf1bc381aa4ead712
+  metadata.gz: 8023ee9f3e35c8192af2ef0cd4231b635b5efc6e5529fe85acb42a641032b3b0
+  data.tar.gz: 2875b07c0f94091745d83cea071d96bcffe1a77b5f493eda45970bc7ff3c580b
 SHA512:
-  metadata.gz: 8305b0f43fec6a78fc64b2d79fde688d829bacb3e4ce772d14a6c68ceaa71141a6948c4d7c1d6a358f21c4e86e601f44c4e6411a0e5c7c2522a0a8fed71856f1
-  data.tar.gz: 6d9418c471d3966f00796dd0840cf33400188e93e61585b0b777cc31db70d020b92cd7f89f00754734a5331db29600ec4955b2b52bc86d62da3c11c98f71b45d
+  metadata.gz: fae6f061d8fb94d44671e372d802190d205058073a389534241f7958aafe0a0c49e87e75e46cd205d4c9ea227e5c4610f9f4ca430849c084a2493eacdca8419f
+  data.tar.gz: a7fa6dcb6a2b379bbc3e6b8de6b4d3f300c0b5e01f40dc79fa784a535f6041d29913b4c904adc3a98e7c8d90ba88ff008e4700549b6c5d397242b33a85040092

data/lib/metanorma/collection/collection.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require "relaton"
 require "relaton/cli"
 require "metanorma-utils"
@@ -28,16 +26,20 @@ module Metanorma
     def initialize(**args)
       @file = args[:file]
       @dirname = File.expand_path(File.dirname(@file)) # feeds @manifest
-      @documents = args[:documents] || {} # feeds initialize_directives
+      @documents = args[:documents] || {} # feeds initialize_directives, initialize_docs
       @bibdatas = args[:documents] || {}
       initialize_vars
       initialize_config(args[:config])
       initialize_directives
+      initialize_docs
+      validate_flavor(flavor)
+    end
+
+    def initialize_docs
       @documents.merge! @manifest.documents
       @bibdatas.merge! @manifest.documents
       @documents.transform_keys { |k| Util::key(k) }
       @bibdatas.transform_keys { |k| Util::key(k) }
-      validate_flavor(flavor)
     end
 
     def initialize_vars
@@ -54,6 +56,8 @@ module Metanorma
       @final = config.final_content
       @manifest = ::Metanorma::Collection::Manifest
         .new(config.manifest, self, @dirname) # feeds initialize_directives
+      @format = config.format.map(&:to_sym)
+      @format&.empty? and @format = nil
     end
 
     def initialize_directives
@@ -88,7 +92,9 @@ module Metanorma
     end
 
     def render(opts)
-      opts[:format].nil? || opts[:format].empty? and opts[:format] = [:html]
+      if opts[:format].nil? || opts[:format].empty?
+        opts[:format] = @format || [:html]
+      end
       opts[:log] = @log
       opts[:flavor] = @flavor
       ::Metanorma::Collection::Renderer.render self, opts
@@ -120,7 +126,14 @@ module Metanorma
     # @return [String] XML
     def sections(cnt)
       c = Asciidoctor.convert(cnt, backend: flavor.to_sym, header_footer: true)
-      Nokogiri::XML(c, &:huge).at("//xmlns:sections").children.to_xml
+      x = Nokogiri::XML(c)
+      x.xpath("//xmlns:clause").each { |n| n["unnumbered"] = true }
+      file = Tempfile.new(%w(foo presentation.xml))
+      file.write(x.to_xml(indent: 0))
+      file.close
+      c1 = Util::isodoc_create(@flavor, @manifest.lang, @manifest.script, x, presxml: true)
+        .convert(file.path, nil, true)
+      Nokogiri::XML(c1).at("//xmlns:sections").children.to_xml
     end
 
     # @param builder [Nokogiri::XML::Builder]

data/lib/metanorma/collection/config/bibdata.rb

@@ -1,10 +1,10 @@
 require "relaton"
-require "shale"
+require "lutaml/model"
 
 module Metanorma
   class Collection
     module Config
-      class Bibdata < ::Shale::Mapper
+      class Bibdata < ::Lutaml::Model::Serializable
         model ::RelatonBib::BibliographicItem
       end
     end

data/lib/metanorma/collection/config/compile_options.rb

@@ -1,12 +1,11 @@
-require "shale"
+require "lutaml/model"
 
 module Metanorma
   class Collection
     module Config
-      class CompileOptions < ::Shale::Mapper
-        attribute :install_fonts, ::Shale::Type::Boolean,
-                  default: -> { false }
-        attribute :agree_to_terms, ::Shale::Type::Boolean, default: -> { true }
+      class CompileOptions < ::Lutaml::Model::Serializable
+        attribute :install_fonts, :boolean, default: -> { false }
+        attribute :agree_to_terms, :boolean, default: -> { true }
       end
     end
   end

data/lib/metanorma/collection/config/config.rb

@@ -1,5 +1,4 @@
-require "shale"
-require_relative "../../shale_monkeypatch"
+require "lutaml/model"
 require_relative "../../array_monkeypatch"
 require_relative "compile_options"
 require_relative "converters"
@@ -11,36 +10,35 @@ module Metanorma
   class Collection
     module Config
       require "shale/adapter/nokogiri"
-      ::Shale.xml_adapter = ::Shale::Adapter::Nokogiri
+      Lutaml::Model::Config.configure do |config|
+        config.xml_adapter = Lutaml::Model::XmlAdapter::NokogiriAdapter
+      end
 
-      class Config < ::Shale::Mapper
+      class Config < ::Lutaml::Model::Serializable
         attr_accessor :path, :collection, :from_xml
 
         attribute :bibdata, Bibdata
         attribute :directive, Directive, collection: true
         attribute :manifest, Manifest
-        attribute :format, ::Shale::Type::String, collection: true,
-                                                  default: -> { [:html] }
-        attribute :output_folder, ::Shale::Type::String
-        attribute :coverpage, ::Shale::Type::String, default: -> {
-                                                                "cover.html"
-                                                              }
+        attribute :coverpage, :string, default: -> { "cover.html" }
+        attribute :format, :string, collection: true, default: -> { [:html] }
+        attribute :output_folder, :string
         attribute :compile, CompileOptions
-        attribute :prefatory_content, ::Shale::Type::String
-        attribute :final_content, ::Shale::Type::String
+        attribute :prefatory_content, :string, raw: true
+        attribute :final_content, :string, raw: true
         attribute :documents, Bibdata, collection: true
-        attribute :xmlns, ::Shale::Type::String, default: -> { "http://metanorma.org" }
+        attribute :xmlns, :string, default: -> { "http://metanorma.org" }
 
         yaml do
-          map "directives", using: { from: :directives_from_yaml,
-                                     to: :directives_to_yaml }
-          map "bibdata", using: { from: :bibdata_from_yaml,
-                                  to: :bibdata_to_yaml }
+          map "directives", to: :directive, with: { from: :directives_from_yaml,
+                                                    to: :directives_to_yaml }
+          map "bibdata", to: :bibdata, with: { from: :bibdata_from_yaml,
+                                               to: :bibdata_to_yaml }
           map "manifest", to: :manifest
-          map "format", to: :format
+          map "format", to: :format, render_default: true
           map "output_folder", to: :output_folder
-          map "coverpage", to: :coverpage
-          map "compile", to: :compile
+          map "coverpage", to: :coverpage, render_default: true
+          map "compile", to: :compile, render_default: true
           map "prefatory-content", to: :prefatory_content
           map "final-content", to: :final_content
         end
@@ -49,27 +47,31 @@ module Metanorma
           root "metanorma-collection"
           # namespace "http://metanorma.org", "m"
           # map_attribute "xmlns", to: :xmlns
-          map_element "bibdata", using: { from: :bibdata_from_xml,
-                                          to: :bibdata_to_xml }
-          map_element "directive", using: { from: :directive_from_xml,
-                                            to: :directive_to_xml }
-          map_element "entry", using: { from: :manifest_from_xml,
-                                        to: :manifest_to_xml }
-          map_element "format", to: :format
+          map_element "bibdata", to: :bibdata, with: { from: :bibdata_from_xml,
+                                                       to: :bibdata_to_xml }
+          map_element "directive", to: :directive
+          map_element "entry", to: :manifest, with: { from: :manifest_from_xml,
+                                                      to: :manifest_to_xml }
+          map_element "format", to: :format, render_default: true
           map_element "output_folder", to: :output_folder
-          map_element "coverpage", to: :coverpage
-          map_element "compile", to: :compile
-          map_element "prefatory-content", using: { from: :prefatory_from_xml,
-                                                    to: :prefatory_to_xml }
+          map_element "coverpage", to: :coverpage, render_default: true
+          map_element "compile", to: :compile, render_default: true
+          map_element "prefatory-content",
+                      to: :prefatory_content,
+                      with: { from: :prefatory_from_xml,
+                              to: :prefatory_to_xml }
           map_element "doc-container",
-                      using: { from: :documents_from_xml,
-                               to: :documents_to_xml }
-          map_element "final-content", using: { from: :final_from_xml,
-                                                to: :final_to_xml }
+                      to: :documents,
+                      with: { from: :documents_from_xml,
+                              to: :documents_to_xml }
+          map_element "final-content",
+                      to: :final_content,
+                      with: { from: :final_from_xml,
+                              to: :final_to_xml }
         end
 
         def manifest_from_xml(model, node)
-          model.manifest = Manifest.from_xml(node.to_xml)
+          model.manifest = Manifest.from_xml(node.node.to_xml)
         end
 
         def manifest_to_xml(model, parent, doc)
@@ -78,7 +80,7 @@ module Metanorma
         end
 
         def prefatory_from_xml(model, node)
-          model.prefatory_content = node.to_xml
+          model.prefatory_content = node
         end
 
         def prefatory_to_xml(model, parent, doc)
@@ -92,7 +94,8 @@ module Metanorma
         def content_to_xml(model, parent, doc, type)
           x = model.send("#{type}_content") or return
           n = Nokogiri::XML(x)
-          elem = if n.elements.size == 1 then n.root
+          elem = if n.elements.size == 1
+                   "<#{type}-content>#{x}</#{type}-content>" # n.root
                  else
                    b = Nokogiri::XML::Builder.new
                    model.collection.content_to_xml(type, b)
@@ -102,19 +105,7 @@ module Metanorma
         end
 
         def final_from_xml(model, node)
-          model.final_content = node.to_xml
-        end
-
-        def directive_from_xml(model, node)
-          model.directive ||= []
-          model.directive << Directive.from_xml(node.to_xml)
-        end
-
-        def directive_to_xml(model, parent, doc)
-          Array(model.directive).each do |e|
-            elem = e.to_xml
-            doc.add_element(parent, elem)
-          end
+          model.final_content = node
         end
 
         def directives_from_yaml(model, value)
@@ -134,28 +125,6 @@ module Metanorma
           end
         end
 
-        def documents_from_xml(model, value)
-          x = if value.is_a?(Shale::Adapter::Nokogiri::Node)
-                value.content
-              else Nokogiri::XML(value)
-              end
-          model.documents = x.xpath(".//bibdata")
-            .each_with_object([]) do |b, m|
-            m << Bibdata.from_xml(b.to_xml)
-          end
-        end
-
-        def documents_to_xml(model, parent, doc)
-          b = Nokogiri::XML::Builder.new do |xml|
-            xml.document do |m|
-              model.collection.doccontainer(m) or return
-            end
-          end
-          b.parent.elements.first.elements.each do |x|
-            doc.add_element(parent, x)
-          end
-        end
-
         include Converters
       end
     end

data/lib/metanorma/collection/config/converters.rb

@@ -14,7 +14,8 @@ module Metanorma
         end
 
         def bibdata_from_xml(model, node)
-          model.bibdata = Relaton::Cli.parse_xml(node.content)
+          node and
+            model.bibdata = Relaton::Cli.parse_xml(node.node.adapter_node)
         end
 
         def bibdata_to_xml(model, parent, doc)
@@ -24,6 +25,31 @@ module Metanorma
         end
 
         def nop_to_yaml(model, doc); end
+
+        def documents_from_xml(model, value)
+          model.documents = value
+            .each_with_object([]) do |b, m|
+            m << b
+          end
+        end
+
+        def documents_to_xml(model, parent, doc)
+          documents_to_xml?(doc) or return
+          b = Nokogiri::XML::Builder.new do |xml|
+            xml.document do |m|
+              model.collection.doccontainer(m) or return
+            end
+          end
+          b.parent.elements.first.elements
+            .each { |x| doc.add_element(parent, x) }
+        end
+
+        def documents_to_xml?(doc)
+          ret = doc.parent.elements.detect do |x|
+            x.name == "doc-container"
+          end
+          !ret
+        end
       end
     end
   end

data/lib/metanorma/collection/config/directive.rb

@@ -1,9 +1,15 @@
 module Metanorma
   class Collection
     module Config
-      class Directive < ::Shale::Mapper
-        attribute :key, ::Shale::Type::String
-        attribute :value, ::Shale::Type::String
+      class Directive < ::Lutaml::Model::Serializable
+        attribute :key, :string
+        attribute :value, :string
+
+        xml do
+          root "directive"
+          map_element "key", to: :key
+          map_element "value", to: :value
+        end
       end
     end
   end

data/lib/metanorma/collection/config/manifest.rb

@@ -1,5 +1,5 @@
-require "shale"
-require_relative "../../shale_monkeypatch"
+require "lutaml/model"
+require "lutaml/model/xml_adapter/nokogiri_adapter"
 require_relative "../../array_monkeypatch"
 require_relative "converters"
 require_relative "bibdata"
@@ -8,58 +8,65 @@ module Metanorma
   class Collection
     module Config
       require "shale/adapter/nokogiri"
-      ::Shale.xml_adapter = ::Shale::Adapter::Nokogiri
+      Lutaml::Model::Config.configure do |config|
+        config.xml_adapter = Lutaml::Model::XmlAdapter::NokogiriAdapter
+      end
 
-      class Manifest < ::Shale::Mapper
-        attribute :identifier, ::Shale::Type::String,
-                  default: -> { UUIDTools::UUID.random_create.to_s }
-        attribute :id, ::Shale::Type::String
+      class Manifest < ::Lutaml::Model::Serializable
+        attribute :identifier, :string,
+          default: -> { UUIDTools::UUID.random_create.to_s }
+        attribute :id, :string
         attribute :bibdata, Bibdata
-        attribute :type, ::Shale::Type::String
-        attribute :title, ::Shale::Type::String
-        attribute :url, ::Shale::Type::String
-        attribute :attachment, ::Shale::Type::Boolean
-        attribute :sectionsplit, ::Shale::Type::Boolean
-        attribute :index, ::Shale::Type::Boolean, default: -> { true }
+        attribute :type, :string
+        attribute :title, :string
+        attribute :url, :string
+        attribute :level, :string
+        attribute :attachment, :boolean
+        attribute :sectionsplit, :boolean
+        attribute :index, :boolean, default: -> { true }
         attribute :entry, Manifest, collection: true
-        attribute :file, ::Shale::Type::String
+        attribute :file, :string
 
         yaml do
           map "identifier", to: :identifier
           map "type", to: :type
-          map "level", using: { from: :level_from_yaml, to: :nop_to_yaml }
+          map "level", to: :level,
+                       with: { from: :level_from_yaml, to: :nop_to_yaml }
           map "title", to: :title
           map "url", to: :url
           map "attachment", to: :attachment
           map "sectionsplit", to: :sectionsplit
           map "index", to: :index
           map "file", to: :file
-          map "fileref", using: { from: :fileref_from_yaml, to: :nop_to_yaml }
+          map "fileref", to: :file,
+                         with: { from: :fileref_from_yaml, to: :nop_to_yaml }
           map "entry", to: :entry
-          map "docref", using: { from: :docref_from_yaml, to: :nop_to_yaml }
-          map "manifest", using: { from: :docref_from_yaml, to: :nop_to_yaml }
-          map "bibdata", using: { from: :bibdata_from_yaml,
-                                  to: :bibdata_to_yaml }
+          map "docref", to: :entry,
+                        with: { from: :docref_from_yaml, to: :nop_to_yaml }
+          map "manifest", to: :entry,
+                          with: { from: :docref_from_yaml, to: :nop_to_yaml }
+          map "bibdata", to: :bibdata, with: { from: :bibdata_from_yaml,
+                                               to: :bibdata_to_yaml }
         end
 
         xml do
           root "entry"
-          map_attribute "id", to: :id
+          map_attribute "target", to: :id
           map_attribute "attachment", to: :attachment
           map_attribute "sectionsplit", to: :sectionsplit
           map_attribute "index", to: :index
           map_attribute "url", to: :url
           map_attribute "fileref", to: :file
-          map_element "identifier", to: :identifier
+          map_element "identifier", to: :identifier, render_default: true
           map_element "type", to: :type
           map_element "title", to: :title
-          map_element "bibdata", using: { from: :bibdata_from_xml,
-                                          to: :bibdata_to_xml }
-          map_element "entry", to: :entry # using: { from: :entry_from_xml, to: :entry_to_xml }
+          map_element "bibdata", to: :bibdata, with: { from: :bibdata_from_xml,
+                                                       to: :bibdata_to_xml }
+          map_element "entry", to: :entry
         end
 
         def entry_from_xml(model, node)
-          model.entry = Manifest.from_xml(node.content)
+          model.entry = Manifest.from_xml(node.node.to_xml)
         end
 
         def entry_to_xml(model, parent, doc)

data/lib/metanorma/collection/filelookup/filelookup.rb

@@ -62,12 +62,24 @@ module Metanorma
           file, _filename = targetfile(entry, read: true)
           xml = Nokogiri::XML(file, &:huge)
           add_document_suffix(ident, xml)
-          entry.merge!(anchors: read_anchors(xml), ids: read_ids(xml),
-                       bibdata: xml.at(ns("//bibdata")),
-                       document_suffix: xml.root["document_suffix"])
+          entry.merge!(bibdata_extract(xml))
         end
       end
 
+      def anchors_lookup(anchors)
+        anchors.values.each_with_object({}) do |v, m|
+          v.each_value { |v1| m[v1] = true }
+        end
+      end
+
+      def bibdata_extract(xml)
+        anchors = read_anchors(xml)
+        { anchors: anchors, anchors_lookup: anchors_lookup(anchors),
+          ids: read_ids(xml),
+          bibdata: xml.at(ns("//bibdata")),
+          document_suffix: xml.root["document_suffix"] }
+      end
+
       def bibitem_process(entry)
         entry[:bibitem] = entry[:bibdata].dup
         entry[:bibitem].name = "bibitem"
@@ -76,7 +88,7 @@ module Metanorma
       end
 
       # ref is the absolute source file address
-      # rel_path is the relative source file address, relative to the YAML locaton
+      # rel_path is the relative source file address, relative to the YAML location
       # out_path is the destination file address, with any references outside
       # the working directory (../../...) truncated, and based on relative path
       # identifier is the id with only spaces, no nbsp
@@ -198,10 +210,10 @@ module Metanorma
         ret[val[:type]] ||= {}
         index = if val[:container] || val[:label].nil? || val[:label].empty?
                   UUIDTools::UUID.random_create.to_s
-                else val[:label]
+                else val[:label].gsub(%r{<[^>]+>}, "")
                 end
         ret[val[:type]][index] = key
-        ret[val[:type]][val[:value]] = key if val[:value]
+        v = val[:value] and ret[val[:type]][v.gsub(%r{<[^>]+>}, "")] = key
       end
 
       # Also parse all ids in doc (including ones which won't be xref targets)
@@ -216,8 +228,8 @@ module Metanorma
       end
 
       def key(ident)
-        @c.decode(ident).gsub(/(\p{Zs})+/, " ").sub(/^metanorma-collection /,
-                                                    "")
+        @c.decode(ident).gsub(/(\p{Zs})+/, " ")
+          .sub(/^metanorma-collection /, "")
       end
 
       def keys

data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb

@@ -1,4 +1,5 @@
 require_relative "../sectionsplit/sectionsplit"
+# require "concurrent-ruby"
 
 module Metanorma
   class Collection
@@ -16,13 +17,25 @@ module Metanorma
 
       def process_section_split_instance(key, manifest)
         s, sectionsplit_manifest = sectionsplit(key)
+        # section_split_instance_threads(s, manifest, key)
         s.each_with_index do |f1, i|
           add_section_split_instance(f1, manifest, key, i)
         end
         a = add_section_split_attachments(sectionsplit_manifest, key) and
           manifest["#{key}:attachments"] = a
-        manifest["#{key}:index.html"] =
-          add_section_split_cover(sectionsplit_manifest, key)
+        add_section_split_cover(manifest, sectionsplit_manifest, key)
+      end
+
+      def section_split_instance_threads(s, manifest, key)
+        @mutex = Mutex.new
+        pool = Concurrent::FixedThreadPool.new(4)
+        s.each_with_index do |f1, i|
+          pool.post do
+            add_section_split_instance(f1, manifest, key, i)
+          end
+        end
+        pool.shutdown
+        pool.wait_for_termination
       end
 
       def cleanup_section_split_instance(key, manifest)
@@ -31,24 +44,34 @@ module Metanorma
         @files[key][:indirect_key] = @sectionsplit.key
       end
 
-      def add_section_split_cover(manifest, ident)
+      def add_section_split_cover(manifest, sectionsplit_manifest, ident)
         cover = @sectionsplit
-          .section_split_cover(manifest, @parent.dir_name_cleanse(ident),
+          .section_split_cover(sectionsplit_manifest,
+                               @parent.dir_name_cleanse(ident),
                                one_doc_collection?)
         @files[ident][:out_path] = cover
-        { attachment: true, index: false, out_path: cover,
-          ref: File.join(File.dirname(manifest.file), cover) }
+        src = File.join(File.dirname(sectionsplit_manifest.file), cover)
+        m = { attachment: true, index: false, out_path: cover, ref: src }
+        manifest["#{ident}:index.html"] = m
+        one_doc_collection? and
+          add_cover_one_doc_coll(manifest, sectionsplit_manifest, ident, m)
+      end
+
+      def add_cover_one_doc_coll(manifest, sectionsplit_manifest, key, entry)
+        idx = File.join(File.dirname(sectionsplit_manifest.file), "index.html")
+        FileUtils.cp entry[:ref], idx
+        manifest["#{key}:index1.html"] =
+          entry.merge(out_path: "index.html", ref: idx)
       end
 
       def one_doc_collection?
-        return false
         docs = 0
         @files.each_value do |v|
           v[:attachment] and next
           v[:presentationxml] and next
           docs += 1
         end
-        docs > 1
+        docs <= 1
       end
 
       def add_section_split_attachments(manifest, ident)
@@ -62,15 +85,16 @@ module Metanorma
 
       def add_section_split_instance(file, manifest, key, idx)
         presfile, newkey, xml = add_section_split_instance_prep(file, key)
-        manifest[newkey] =
-          { parentid: key, presentationxml: true, type: "fileref",
-            rel_path: file[:url], out_path: File.basename(file[:url]),
-            anchors: read_anchors(xml), ids: read_ids(xml),
-            sectionsplit_output: true,
-            bibdata: @files[key][:bibdata], ref: presfile }
+        anchors = read_anchors(xml)
+        m = { parentid: key, presentationxml: true, type: "fileref",
+              rel_path: file[:url], out_path: File.basename(file[:url]),
+              anchors: anchors, anchors_lookup: anchors_lookup(anchors),
+              ids: read_ids(xml),
+              sectionsplit_output: true, indirect_key: @sectionsplit.key,
+              bibdata: @files[key][:bibdata], ref: presfile }
+        m[:bare] = true unless idx.zero?
+        manifest[newkey] = m
         @files_to_delete << file[:url]
-        manifest[newkey][:indirect_key] = @sectionsplit.key
-        manifest[newkey][:bare] = true unless idx.zero?
       end
 
       def add_section_split_instance_prep(file, key)

data/lib/metanorma/collection/manifest/manifest.rb

@@ -6,7 +6,7 @@ module Metanorma
   class Collection
     class Manifest
       # @return [Metanorma::Collection]
-      attr_reader :collection, :config
+      attr_reader :collection, :config, :lang, :script
 
       # @param level [String]
       # @param dir [String]