RubyGems - metanorma - Versions diffs - 2.0.5 → 2.0.7 - Mend

metanorma 2.0.5 → 2.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (25) hide show

checksums.yaml +4 -4
data/lib/metanorma/collection/collection.rb +32 -12
data/lib/metanorma/collection/config/manifest.rb +2 -2
data/lib/metanorma/collection/document/document.rb +12 -10
data/lib/metanorma/collection/filelookup/filelookup.rb +19 -7
data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb +46 -20
data/lib/metanorma/collection/manifest/manifest.rb +13 -8
data/lib/metanorma/collection/multilingual/multilingual.rb +2 -2
data/lib/metanorma/collection/renderer/fileparse.rb +98 -97
data/lib/metanorma/collection/renderer/fileprocess.rb +1 -0
data/lib/metanorma/collection/renderer/render_word.rb +6 -6
data/lib/metanorma/collection/renderer/renderer.rb +9 -12
data/lib/metanorma/collection/renderer/svg.rb +44 -0
data/lib/metanorma/collection/renderer/utils.rb +22 -6
data/lib/metanorma/collection/sectionsplit/collection.rb +100 -0
data/lib/metanorma/collection/sectionsplit/sectionsplit.rb +73 -101
data/lib/metanorma/collection/util/util.rb +3 -4
data/lib/metanorma/collection/xrefprocess/xrefprocess.rb +3 -3
data/lib/metanorma/compile/compile.rb +9 -0
data/lib/metanorma/compile/compile_validate.rb +4 -4
data/lib/metanorma/compile/extract.rb +2 -4
data/lib/metanorma/input/asciidoc.rb +11 -30
data/lib/metanorma/version.rb +1 -1
data/metanorma.gemspec +2 -1
metadata +20 -4

data/lib/metanorma/collection/sectionsplit/sectionsplit.rb CHANGED Viewed

@@ -1,6 +1,8 @@
 require "yaml"
 require_relative "../../util/util"
 require_relative "../xrefprocess/xrefprocess"
+require_relative "collection"
+require "concurrent-ruby"
 module Metanorma
   class Collection
@@ -17,49 +19,13 @@ module Metanorma
         @fileslookup = opts[:fileslookup]
         @ident = opts[:ident]
         @isodoc = opts[:isodoc]
+        @document_suffix = opts[:document_suffix]
       end
       def ns(xpath)
         @isodoc.ns(xpath)
       end
-      def build_collection
-        collection_setup(@base, @dir)
-        files = sectionsplit # (@input_filename, @base, @dir, @compile_opts)
-        input_xml = Nokogiri::XML(File.read(@input_filename,
-                                            encoding: "UTF-8"), &:huge)
-        collection_manifest(@base, files, input_xml, @xml, @dir).render(
-          { format: %i(html), output_folder: "#{@output_filename}_collection",
-            coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
-        )
-        section_split_attachments(out: "#{@output_filename}_collection")
-      end
-      def collection_manifest(filename, files, origxml, _presxml, dir)
-        File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
-          f.write(collectionyaml(files, origxml))
-        end
-        Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
-      end
-      def collection_setup(filename, dir)
-        FileUtils.mkdir_p "#{filename}_collection" if filename
-        FileUtils.mkdir_p dir
-        File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
-          f.write(coll_cover)
-        end
-      end
-      def coll_cover
-        <<~COVER
-          <html><head><meta charset="UTF-8"/></head><body>
-                <h1>{{ doctitle }}</h1>
-                <h2>{{ docnumber }}</h2>
-                <nav>{{ navigation }}</nav>
-              </body></html>
-        COVER
-      end
       SPLITSECTIONS =
         [["//preface/*", "preface"], ["//sections/*", "sections"],
          ["//annex", nil],
@@ -70,17 +36,37 @@ module Metanorma
       def sectionsplit
         xml = sectionsplit_prep(File.read(@input_filename), @base, @dir)
         @key = Metanorma::Collection::XrefProcess::xref_preprocess(xml, @isodoc)
-        SPLITSECTIONS.each_with_object([]) do |n, ret|
+        empty = empty_doc(xml)
+        empty1 = empty_attachments(empty)
+        @mutex = Mutex.new
+        @pool = Concurrent::FixedThreadPool.new(4)
+        sectionsplit1(xml, empty, empty1, 0)
+      end
+      def sectionsplit1(xml, empty, empty1, idx)
+        ret = SPLITSECTIONS.each_with_object([]) do |n, m|
           conflate_floatingtitles(xml.xpath(ns(n[0]))).each do |s|
-            ret << sectionfile(xml, emptydoc(xml, ret.size),
-                               "#{@base}.#{ret.size}", s, n[1])
+            sectionsplit2(xml, idx.zero? ? empty : empty1, s, n[1],
+                          { acc: m, idx: idx })
+            idx += 1
           end
         end
+        @pool.shutdown
+        @pool.wait_for_termination
+        ret
+      end
+      def sectionsplit2(xml, empty, chunks, parentnode, opt)
+        @pool.post do
+          a = sectionfile(xml, empty, "#{@base}.#{opt[:idx]}", chunks,
+                          parentnode)
+          @mutex.synchronize { opt[:acc] << a }
+        end
       end
       def block?(node)
         %w(p table formula admonition ol ul dl figure quote sourcecode example
-           pre note pagebrreak hr bookmark requirement recommendation permission
+           pre note pagebreak hr bookmark requirement recommendation permission
            svgmap inputform toc passthrough review imagemap).include?(node.name)
       end
@@ -142,7 +128,26 @@ module Metanorma
              "//bibliography/references[not(@hidden = 'true')] | " \
              "//indexsect | //colophon"),
         ).each(&:remove)
-        ordinal.zero? or out.xpath(ns("//metanorma-ext//attachment"))
+        ordinal.zero? or out.xpath(ns("//metanorma-ext//attachment | " \
+                                      "//semantic__metanorma-ext//semantic__attachment"))
+          .each(&:remove) # keep only one copy of attachments
+        out
+      end
+      def empty_doc(xml)
+        out = xml.dup
+        out.xpath(
+          ns("//preface | //sections | //annex | //bibliography/clause | " \
+             "//bibliography/references[not(@hidden = 'true')] | " \
+             "//indexsect | //colophon"),
+        ).each(&:remove)
+        out
+      end
+      def empty_attachments(xml)
+        out = xml.dup
+        out.xpath(ns("//metanorma-ext//attachment | " \
+                     "//semantic__metanorma-ext//semantic__attachment"))
           .each(&:remove) # keep only one copy of attachments
         out
       end
@@ -158,11 +163,36 @@ module Metanorma
         sectionfile_insert(ins, chunks, parentnode)
         Metanorma::Collection::XrefProcess::xref_process(out, xml, @key,
                                                          @ident, @isodoc)
+        truncate_semxml(out, chunks)
         outname = "#{file}.xml"
-        File.open(File.join(@splitdir, outname), "w:UTF-8") { |f| f.write(out) }
+        File.open(File.join(@splitdir, outname), "w:UTF-8") do |f|
+          f.write(out)
+        end
         outname
       end
+      def semantic_xml_ids_gather(out)
+        out.at(ns("//semantic__bibdata")) or return
+        SPLITSECTIONS.each_with_object({}) do |s, m|
+          out.xpath(ns(s[0].sub("//", "//semantic__"))).each do |x|
+            x["id"] or next
+            m[x["id"].sub(/^semantic__/, "")] = x
+          end
+        end
+      end
+      def semxml_presxml_nodes_match(nodes, chunks)
+        chunks.each do |x|
+          nodes[x["id"]] and nodes.delete(x["id"])
+        end
+      end
+      def truncate_semxml(out, chunks)
+        nodes = semantic_xml_ids_gather(out) or return
+        semxml_presxml_nodes_match(nodes, chunks)
+        nodes.each_value(&:remove)
+      end
       def sectionfile_insert(ins, chunks, parentnode)
         if parentnode
           ins.next = "<#{parentnode}/>"
@@ -180,64 +210,6 @@ module Metanorma
           .each { |x| x.replace(x.children) }
         t.children.to_xml
       end
-      def collectionyaml(files, xml)
-        ret = {
-          directives: ["presentation-xml", "bare-after-first"],
-          bibdata: {
-            title: {
-              type: "title-main", language: @lang,
-              content: xml.at(ns("//bibdata/title")).text
-            },
-            type: "collection",
-            docid: {
-              type: xml.at(ns("//bibdata/docidentifier/@type")).text,
-              id: xml.at(ns("//bibdata/docidentifier")).text,
-            },
-          },
-          manifest: {
-            level: "collection", title: "Collection",
-            docref: files.sort_by { |f| f[:order] }.each.map do |f|
-              { fileref: f[:url], identifier: f[:title] }
-            end
-          },
-        }
-        ::Metanorma::Util::recursive_string_keys(ret).to_yaml
-      end
-      def section_split_cover(col, ident, _one_doc_coll)
-        dir = File.dirname(col.file)
-        collection_setup(nil, dir)
-        r = ::Metanorma::Collection::Renderer
-          .new(col, dir, output_folder: "#{ident}_collection",
-                         format: %i(html), coverpage: File.join(dir, "cover.html"))
-        r.coverpage
-        section_split_cover1(ident, r, dir, _one_doc_coll)
-      end
-      def att_dir(file)
-        "_#{File.basename(file, '.*')}_attachments"
-      end
-      def section_split_attachments(out: nil)
-        attachments = att_dir(@tmp_filename)
-        File.directory?(attachments) or return
-        dir = out || File.dirname(@input_filename)
-        ret = File.join(dir, att_dir(@output_filename))
-        FileUtils.rm_rf ret
-        FileUtils.mv attachments, ret
-        File.basename(ret)
-      end
-      def section_split_cover1(ident, renderer, dir, _one_doc_coll)
-        # filename = one_doc_coll ? "#{ident}_index.html" : "index.html"
-        filename = File.basename("#{ident}_index.html")
-        # ident can be a directory with YAML indirection
-        FileUtils.mv File.join(renderer.outdir, "index.html"),
-                     File.join(dir, filename)
-        FileUtils.rm_rf renderer.outdir
-        filename
-      end
     end
   end
 end

data/lib/metanorma/collection/util/util.rb CHANGED Viewed

@@ -30,8 +30,7 @@ module Metanorma
           end
         end
-        def add_suffix_to_attributes(doc, suffix, tag_name, attr_name,
-isodoc)
+        def add_suffix_to_attrs(doc, suffix, tag_name, attr_name, isodoc)
           (suffix.nil? || suffix.empty?) and return
           doc.xpath(isodoc.ns("//#{tag_name}[@#{attr_name}]")).each do |elem|
             a = elem.attributes[attr_name].value
@@ -62,8 +61,8 @@ isodoc)
           def attr(_key); end
         end
-        def load_isodoc(doctype)
-          x = Asciidoctor.load nil, backend: doctype.to_sym
+        def load_isodoc(flavor)
+          x = Asciidoctor.load nil, backend: flavor.to_sym
           x.converter.html_converter(Dummy.new) # to obtain Isodoc class
         end
       end

data/lib/metanorma/collection/xrefprocess/xrefprocess.rb CHANGED Viewed

@@ -10,9 +10,9 @@ module Metanorma
           key = (0...8).map { rand(65..90).chr }.join # random string
           xml.root["type"] = key
           Metanorma::Utils::anchor_attributes
-            .each do |(tag_name, attribute_name)|
-            ::Metanorma::Collection::Util::add_suffix_to_attributes(
-              xml, xml.root["document_suffix"], tag_name, attribute_name, isodoc
+            .each do |(tag_name, attr_name)|
+            ::Metanorma::Collection::Util::add_suffix_to_attrs(
+              xml, xml.root["document_suffix"], tag_name, attr_name, isodoc
             )
           end
           key

data/lib/metanorma/compile/compile.rb CHANGED Viewed

@@ -33,6 +33,7 @@ module Metanorma
       relaton_export(isodoc, options)
       extract(isodoc, options[:extract], options[:extract_type])
       process_exts(filename, extensions, file, isodoc, options)
+    ensure
       clean_exit(options)
     end
@@ -113,6 +114,14 @@ module Metanorma
       fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
                  orig_filename: File.expand_path(filename),
                  presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+      if extensions == %i(presentation)
+        process_ext(:presentation, file, isodoc, fnames, options)
+      else
+      process_exts_queue(fnames, extensions, file, isodoc, options)
+      end
+    end
+    def process_exts_queue(fnames, extensions, file, isodoc, options)
       @queue = ::Metanorma::Util::WorkersPool
         .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
       gather_and_install_fonts(file, options.dup, extensions)

data/lib/metanorma/compile/compile_validate.rb CHANGED Viewed

@@ -33,15 +33,15 @@ module Metanorma
                  "support the standard type #{stdtype}. Exiting.", :fatal)
     end
-    private
-    STDTYPE2FLAVOR = {}.freeze
     def stdtype2flavor(stdtype)
       flavor = STDTYPE2FLAVOR[stdtype] || stdtype
       "metanorma-#{flavor}"
     end
+    private
+    STDTYPE2FLAVOR = {}.freeze
     def require_flavor(flavor)
       require flavor
       Util.log("[metanorma] Info: gem `#{flavor}` loaded.", :info)

data/lib/metanorma/compile/extract.rb CHANGED Viewed

@@ -1,8 +1,7 @@
 module Metanorma
   class Compile
     def relaton_export(isodoc, options)
-      return unless options[:relaton]
+      options[:relaton] or return
       xml = Nokogiri::XML(isodoc, &:huge)
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
       # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
@@ -18,8 +17,7 @@ module Metanorma
     end
     def extract(isodoc, dirname, extract_types)
-      return unless dirname
+      dirname or return
       extract_types.nil? || extract_types.empty? and
         extract_types = %i[sourcecode image requirement]
       FileUtils.rm_rf dirname

data/lib/metanorma/input/asciidoc.rb CHANGED Viewed

@@ -10,39 +10,20 @@ module Metanorma
                      novalid: options[:novalid],
                      attributes: ["nodoc", "stem", "docfile=#{filename}",
                                   "output_dir=#{options[:output_dir]}"] }
-        unless asciidoctor_validate(file, filename, out_opts)
-          warn "Cannot continue compiling Asciidoctor document"
-          abort
-        end
         ::Asciidoctor.convert(file, out_opts)
       end
-      def asciidoctor_validate(file, filename, options)
-        err = nil
-        begin
-          previous_stderr = $stderr
-          $stderr = StringIO.new
-          ::Asciidoctor.load(file, options)
-          %r{(\n|^)asciidoctor: ERROR: ['"]?#{Regexp.escape(filename ||
-            '<empty>')}['"]?: line \d+: include file not found: }
-            .match($stderr.string) and err = $stderr.string
-        ensure
-          $stderr = previous_stderr
-        end
-        warn err unless err.nil?
-        err.nil?
-      end
       def extract_metanorma_options(file)
-        headerextract = file.sub(/\n\n.*$/m, "\n")
-        /\n:mn-document-class:\s+(?<type>\S[^\n]*)\n/ =~ headerextract
-        /\n:mn-output-extensions:\s+(?<extensions>\S[^\n]*)\n/ =~ headerextract
-        /\n:mn-relaton-output-file:\s+(?<relaton>\S[^\n]*)\n/ =~ headerextract
-        /\n(?<asciimath>:mn-keep-asciimath:[^\n]*)\n/ =~ headerextract
-        /\n(?<novalid>:novalid:[^\n]*)\n/ =~ headerextract
-        asciimath = if defined?(asciimath)
-                      !asciimath.nil? && asciimath != ":mn-keep-asciimath: false"
-                    end
+        hdr = file.sub(/\n\n.*$/m, "\n")
+        /\n:(?:mn-)?(?:document-class|flavor):\s+(?<type>\S[^\n]*)\n/ =~ hdr
+        /\n:(?:mn-)?output-extensions:\s+(?<extensions>\S[^\n]*)\n/ =~ hdr
+        /\n:(?:mn-)?relaton-output-file:\s+(?<relaton>\S[^\n]*)\n/ =~ hdr
+        /\n(?<asciimath>:(?:mn-)?keep-asciimath:[^\n]*)\n/ =~ hdr
+        /\n(?<novalid>:novalid:[^\n]*)\n/ =~ hdr
+        if defined?(asciimath)
+          asciimath =
+            !asciimath.nil? && !/keep-asciimath: false/.match?(asciimath)
+        end
         asciimath = nil if asciimath == false
         {
           type: defined?(type) ? type&.strip : nil,
@@ -82,7 +63,7 @@ module Metanorma
            toc-tables toc-recommendations).freeze
       def attr_name_normalise(name)
-        name.gsub("-", "").sub(/override$/, "_override").sub(/pdf$/, "_pdf")
+        name.delete("-").sub(/override$/, "_override").sub(/pdf$/, "_pdf")
           .to_sym
       end

data/lib/metanorma/version.rb CHANGED Viewed

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "2.0.5".freeze
+  VERSION = "2.0.7".freeze
 end

data/metanorma.gemspec CHANGED Viewed

@@ -24,10 +24,11 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 3.1.0"
   spec.add_runtime_dependency "asciidoctor"
+  spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "fontist", ">= 1.14.3"
   spec.add_runtime_dependency "htmlentities"
   spec.add_runtime_dependency "isodoc", ">= 2.6.3"
-  spec.add_runtime_dependency "mn2pdf", "~> 1"
+  spec.add_runtime_dependency "mn2pdf", "~> 2"
   spec.add_runtime_dependency "nokogiri"
   spec.add_runtime_dependency "shale"

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.0.7
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-30 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fontist
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -316,7 +330,9 @@ files:
 - lib/metanorma/collection/renderer/navigation.rb
 - lib/metanorma/collection/renderer/render_word.rb
 - lib/metanorma/collection/renderer/renderer.rb
+- lib/metanorma/collection/renderer/svg.rb
 - lib/metanorma/collection/renderer/utils.rb
+- lib/metanorma/collection/sectionsplit/collection.rb
 - lib/metanorma/collection/sectionsplit/sectionsplit.rb
 - lib/metanorma/collection/util/disambig_files.rb
 - lib/metanorma/collection/util/util.rb