RubyGems - metanorma - Versions diffs - 2.0.5 → 2.0.7 - Mend

metanorma 2.0.5 → 2.0.7

Files changed (25) hide show

checksums.yaml +4 -4
data/lib/metanorma/collection/collection.rb +32 -12
data/lib/metanorma/collection/config/manifest.rb +2 -2
data/lib/metanorma/collection/document/document.rb +12 -10
data/lib/metanorma/collection/filelookup/filelookup.rb +19 -7
data/lib/metanorma/collection/filelookup/filelookup_sectionsplit.rb +46 -20
data/lib/metanorma/collection/manifest/manifest.rb +13 -8
data/lib/metanorma/collection/multilingual/multilingual.rb +2 -2
data/lib/metanorma/collection/renderer/fileparse.rb +98 -97
data/lib/metanorma/collection/renderer/fileprocess.rb +1 -0
data/lib/metanorma/collection/renderer/render_word.rb +6 -6
data/lib/metanorma/collection/renderer/renderer.rb +9 -12
data/lib/metanorma/collection/renderer/svg.rb +44 -0
data/lib/metanorma/collection/renderer/utils.rb +22 -6
data/lib/metanorma/collection/sectionsplit/collection.rb +100 -0
data/lib/metanorma/collection/sectionsplit/sectionsplit.rb +73 -101
data/lib/metanorma/collection/util/util.rb +3 -4
data/lib/metanorma/collection/xrefprocess/xrefprocess.rb +3 -3
data/lib/metanorma/compile/compile.rb +9 -0
data/lib/metanorma/compile/compile_validate.rb +4 -4
data/lib/metanorma/compile/extract.rb +2 -4
data/lib/metanorma/input/asciidoc.rb +11 -30
data/lib/metanorma/version.rb +1 -1
data/metanorma.gemspec +2 -1
metadata +20 -4

data/lib/metanorma/collection/sectionsplit/sectionsplit.rb CHANGED Viewed

@@ -1,6 +1,8 @@
 require "yaml"
 require_relative "../../util/util"
 require_relative "../xrefprocess/xrefprocess"
+require_relative "collection"
+require "concurrent-ruby"
 module Metanorma
   class Collection
@@ -17,49 +19,13 @@ module Metanorma
         @fileslookup = opts[:fileslookup]
         @ident = opts[:ident]
         @isodoc = opts[:isodoc]
+        @document_suffix = opts[:document_suffix]
       end
       def ns(xpath)
         @isodoc.ns(xpath)
       end
-      def build_collection
-        collection_setup(@base, @dir)
-        files = sectionsplit # (@input_filename, @base, @dir, @compile_opts)
-        input_xml = Nokogiri::XML(File.read(@input_filename,
-                                            encoding: "UTF-8"), &:huge)
-        collection_manifest(@base, files, input_xml, @xml, @dir).render(
-          { format: %i(html), output_folder: "#{@output_filename}_collection",
-            coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
-        )
-        section_split_attachments(out: "#{@output_filename}_collection")
-      end
-      def collection_manifest(filename, files, origxml, _presxml, dir)
-        File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
-          f.write(collectionyaml(files, origxml))
-        end
-        Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
-      end
-      def collection_setup(filename, dir)
-        FileUtils.mkdir_p "#{filename}_collection" if filename
-        FileUtils.mkdir_p dir
-        File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
-          f.write(coll_cover)
-        end
-      end
-      def coll_cover
-        <<~COVER
-          <html><head><meta charset="UTF-8"/></head><body>
-                <h1>{{ doctitle }}</h1>
-                <h2>{{ docnumber }}</h2>
-                <nav>{{ navigation }}</nav>
-              </body></html>
-        COVER
-      end
       SPLITSECTIONS =
         [["//preface/*", "preface"], ["//sections/*", "sections"],
          ["//annex", nil],
@@ -70,17 +36,37 @@ module Metanorma
       def sectionsplit
         xml = sectionsplit_prep(File.read(@input_filename), @base, @dir)
         @key = Metanorma::Collection::XrefProcess::xref_preprocess(xml, @isodoc)
-        SPLITSECTIONS.each_with_object([]) do |n, ret|
+        empty = empty_doc(xml)
+        empty1 = empty_attachments(empty)
+        @mutex = Mutex.new
+        @pool = Concurrent::FixedThreadPool.new(4)
+        sectionsplit1(xml, empty, empty1, 0)
+      end
+      def sectionsplit1(xml, empty, empty1, idx)
+        ret = SPLITSECTIONS.each_with_object([]) do |n, m|
           conflate_floatingtitles(xml.xpath(ns(n[0]))).each do |s|
-            ret << sectionfile(xml, emptydoc(xml, ret.size),
-                               "#{@base}.#{ret.size}", s, n[1])
+            sectionsplit2(xml, idx.zero? ? empty : empty1, s, n[1],
+                          { acc: m, idx: idx })
+            idx += 1
           end
         end
+        @pool.shutdown
+        @pool.wait_for_termination
+        ret
+      end
+      def sectionsplit2(xml, empty, chunks, parentnode, opt)
+        @pool.post do
+          a = sectionfile(xml, empty, "#{@base}.#{opt[:idx]}", chunks,
+                          parentnode)
+          @mutex.synchronize { opt[:acc] << a }
+        end
       end
       def block?(node)
         %w(p table formula admonition ol ul dl figure quote sourcecode example
-           pre note pagebrreak hr bookmark requirement recommendation permission
+           pre note pagebreak hr bookmark requirement recommendation permission
            svgmap inputform toc passthrough review imagemap).include?(node.name)
       end
@@ -142,7 +128,26 @@ module Metanorma
              "//bibliography/references[not(@hidden = 'true')] | " \
              "//indexsect | //colophon"),
         ).each(&:remove)
-        ordinal.zero? or out.xpath(ns("//metanorma-ext//attachment"))
+        ordinal.zero? or out.xpath(ns("//metanorma-ext//attachment | " \
+                                      "//semantic__metanorma-ext//semantic__attachment"))
+          .each(&:remove) # keep only one copy of attachments
+        out
+      end
+      def empty_doc(xml)
+        out = xml.dup
+        out.xpath(
+          ns("//preface | //sections | //annex | //bibliography/clause | " \
+             "//bibliography/references[not(@hidden = 'true')] | " \
+             "//indexsect | //colophon"),
+        ).each(&:remove)
+        out
+      end
+      def empty_attachments(xml)
+        out = xml.dup
+        out.xpath(ns("//metanorma-ext//attachment | " \
+                     "//semantic__metanorma-ext//semantic__attachment"))
           .each(&:remove) # keep only one copy of attachments
         out
       end
@@ -158,11 +163,36 @@ module Metanorma
         sectionfile_insert(ins, chunks, parentnode)
         Metanorma::Collection::XrefProcess::xref_process(out, xml, @key,
                                                          @ident, @isodoc)
+        truncate_semxml(out, chunks)
         outname = "#{file}.xml"
-        File.open(File.join(@splitdir, outname), "w:UTF-8") { |f| f.write(out) }
+        File.open(File.join(@splitdir, outname), "w:UTF-8") do |f|
+          f.write(out)
+        end
         outname
       end
+      def semantic_xml_ids_gather(out)
+        out.at(ns("//semantic__bibdata")) or return
+        SPLITSECTIONS.each_with_object({}) do |s, m|
+          out.xpath(ns(s[0].sub("//", "//semantic__"))).each do |x|
+            x["id"] or next
+            m[x["id"].sub(/^semantic__/, "")] = x
+          end
+        end
+      end
+      def semxml_presxml_nodes_match(nodes, chunks)
+        chunks.each do |x|
+          nodes[x["id"]] and nodes.delete(x["id"])
+        end
+      end
+      def truncate_semxml(out, chunks)
+        nodes = semantic_xml_ids_gather(out) or return
+        semxml_presxml_nodes_match(nodes, chunks)
+        nodes.each_value(&:remove)
+      end
       def sectionfile_insert(ins, chunks, parentnode)
         if parentnode
           ins.next = "<#{parentnode}/>"
@@ -180,64 +210,6 @@ module Metanorma
           .each { |x| x.replace(x.children) }
         t.children.to_xml
       end
-      def collectionyaml(files, xml)
-        ret = {
-          directives: ["presentation-xml", "bare-after-first"],
-          bibdata: {
-            title: {
-              type: "title-main", language: @lang,
-              content: xml.at(ns("//bibdata/title")).text
-            },
-            type: "collection",
-            docid: {
-              type: xml.at(ns("//bibdata/docidentifier/@type")).text,
-              id: xml.at(ns("//bibdata/docidentifier")).text,
-            },
-          },
-          manifest: {
-            level: "collection", title: "Collection",
-            docref: files.sort_by { |f| f[:order] }.each.map do |f|
-              { fileref: f[:url], identifier: f[:title] }
-            end
-          },
-        }
-        ::Metanorma::Util::recursive_string_keys(ret).to_yaml
-      end
-      def section_split_cover(col, ident, _one_doc_coll)
-        dir = File.dirname(col.file)
-        collection_setup(nil, dir)
-        r = ::Metanorma::Collection::Renderer
-          .new(col, dir, output_folder: "#{ident}_collection",
-                         format: %i(html), coverpage: File.join(dir, "cover.html"))
-        r.coverpage
-        section_split_cover1(ident, r, dir, _one_doc_coll)
-      end
-      def att_dir(file)
-        "_#{File.basename(file, '.*')}_attachments"
-      end
-      def section_split_attachments(out: nil)
-        attachments = att_dir(@tmp_filename)
-        File.directory?(attachments) or return
-        dir = out || File.dirname(@input_filename)
-        ret = File.join(dir, att_dir(@output_filename))
-        FileUtils.rm_rf ret
-        FileUtils.mv attachments, ret
-        File.basename(ret)
-      end
-      def section_split_cover1(ident, renderer, dir, _one_doc_coll)
-        # filename = one_doc_coll ? "#{ident}_index.html" : "index.html"
-        filename = File.basename("#{ident}_index.html")
-        # ident can be a directory with YAML indirection
-        FileUtils.mv File.join(renderer.outdir, "index.html"),
-                     File.join(dir, filename)
-        FileUtils.rm_rf renderer.outdir
-        filename
-      end
     end
   end
 end

data/lib/metanorma/collection/util/util.rb CHANGED Viewed

@@ -30,8 +30,7 @@ module Metanorma
           end
         end
-        def add_suffix_to_attributes(doc, suffix, tag_name, attr_name,
-isodoc)
+        def add_suffix_to_attrs(doc, suffix, tag_name, attr_name, isodoc)
           (suffix.nil? || suffix.empty?) and return
           doc.xpath(isodoc.ns("//#{tag_name}[@#{attr_name}]")).each do |elem|
             a = elem.attributes[attr_name].value
@@ -62,8 +61,8 @@ isodoc)
           def attr(_key); end
         end
-        def load_isodoc(doctype)
-          x = Asciidoctor.load nil, backend: doctype.to_sym
+        def load_isodoc(flavor)
+          x = Asciidoctor.load nil, backend: flavor.to_sym
           x.converter.html_converter(Dummy.new) # to obtain Isodoc class
         end
       end

data/lib/metanorma/collection/xrefprocess/xrefprocess.rb CHANGED Viewed

@@ -10,9 +10,9 @@ module Metanorma
           key = (0...8).map { rand(65..90).chr }.join # random string
           xml.root["type"] = key
           Metanorma::Utils::anchor_attributes
-            .each do |(tag_name, attribute_name)|
-            ::Metanorma::Collection::Util::add_suffix_to_attributes(
-              xml, xml.root["document_suffix"], tag_name, attribute_name, isodoc
+            .each do |(tag_name, attr_name)|
+            ::Metanorma::Collection::Util::add_suffix_to_attrs(
+              xml, xml.root["document_suffix"], tag_name, attr_name, isodoc
             )
           end
           key

data/lib/metanorma/compile/compile.rb CHANGED Viewed

@@ -33,6 +33,7 @@ module Metanorma
       relaton_export(isodoc, options)
       extract(isodoc, options[:extract], options[:extract_type])
       process_exts(filename, extensions, file, isodoc, options)
+    ensure
       clean_exit(options)
     end
@@ -113,6 +114,14 @@ module Metanorma
       fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
                  orig_filename: File.expand_path(filename),
                  presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+      if extensions == %i(presentation)
+        process_ext(:presentation, file, isodoc, fnames, options)
+      else
+      process_exts_queue(fnames, extensions, file, isodoc, options)
+      end
+    end
+    def process_exts_queue(fnames, extensions, file, isodoc, options)
       @queue = ::Metanorma::Util::WorkersPool
         .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
       gather_and_install_fonts(file, options.dup, extensions)

data/lib/metanorma/compile/compile_validate.rb CHANGED Viewed

@@ -33,15 +33,15 @@ module Metanorma
                  "support the standard type #{stdtype}. Exiting.", :fatal)
     end
-    private
-    STDTYPE2FLAVOR = {}.freeze
     def stdtype2flavor(stdtype)
       flavor = STDTYPE2FLAVOR[stdtype] || stdtype
       "metanorma-#{flavor}"
     end
+    private
+    STDTYPE2FLAVOR = {}.freeze
     def require_flavor(flavor)
       require flavor
       Util.log("[metanorma] Info: gem `#{flavor}` loaded.", :info)

data/lib/metanorma/compile/extract.rb CHANGED Viewed

@@ -1,8 +1,7 @@
 module Metanorma
   class Compile
     def relaton_export(isodoc, options)
-      return unless options[:relaton]
+      options[:relaton] or return
       xml = Nokogiri::XML(isodoc, &:huge)
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
       # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
@@ -18,8 +17,7 @@ module Metanorma
     end
     def extract(isodoc, dirname, extract_types)
-      return unless dirname
+      dirname or return
       extract_types.nil? || extract_types.empty? and
         extract_types = %i[sourcecode image requirement]
       FileUtils.rm_rf dirname

data/lib/metanorma/input/asciidoc.rb CHANGED Viewed

@@ -10,39 +10,20 @@ module Metanorma
                      novalid: options[:novalid],
                      attributes: ["nodoc", "stem", "docfile=#{filename}",
                                   "output_dir=#{options[:output_dir]}"] }
-        unless asciidoctor_validate(file, filename, out_opts)
-          warn "Cannot continue compiling Asciidoctor document"
-          abort
-        end
         ::Asciidoctor.convert(file, out_opts)
       end
-      def asciidoctor_validate(file, filename, options)
-        err = nil
-        begin
-          previous_stderr = $stderr
-          $stderr = StringIO.new
-          ::Asciidoctor.load(file, options)
-          %r{(\n|^)asciidoctor: ERROR: ['"]?#{Regexp.escape(filename ||
-            '<empty>')}['"]?: line \d+: include file not found: }
-            .match($stderr.string) and err = $stderr.string
-        ensure
-          $stderr = previous_stderr
-        end
-        warn err unless err.nil?
-        err.nil?
-      end
       def extract_metanorma_options(file)
-        headerextract = file.sub(/\n\n.*$/m, "\n")
-        /\n:mn-document-class:\s+(?<type>\S[^\n]*)\n/ =~ headerextract
-        /\n:mn-output-extensions:\s+(?<extensions>\S[^\n]*)\n/ =~ headerextract
-        /\n:mn-relaton-output-file:\s+(?<relaton>\S[^\n]*)\n/ =~ headerextract
-        /\n(?<asciimath>:mn-keep-asciimath:[^\n]*)\n/ =~ headerextract
-        /\n(?<novalid>:novalid:[^\n]*)\n/ =~ headerextract
-        asciimath = if defined?(asciimath)
-                      !asciimath.nil? && asciimath != ":mn-keep-asciimath: false"
-                    end
+        hdr = file.sub(/\n\n.*$/m, "\n")
+        /\n:(?:mn-)?(?:document-class|flavor):\s+(?<type>\S[^\n]*)\n/ =~ hdr
+        /\n:(?:mn-)?output-extensions:\s+(?<extensions>\S[^\n]*)\n/ =~ hdr
+        /\n:(?:mn-)?relaton-output-file:\s+(?<relaton>\S[^\n]*)\n/ =~ hdr
+        /\n(?<asciimath>:(?:mn-)?keep-asciimath:[^\n]*)\n/ =~ hdr
+        /\n(?<novalid>:novalid:[^\n]*)\n/ =~ hdr
+        if defined?(asciimath)
+          asciimath =
+            !asciimath.nil? && !/keep-asciimath: false/.match?(asciimath)
+        end
         asciimath = nil if asciimath == false
         {
           type: defined?(type) ? type&.strip : nil,
@@ -82,7 +63,7 @@ module Metanorma
            toc-tables toc-recommendations).freeze
       def attr_name_normalise(name)
-        name.gsub("-", "").sub(/override$/, "_override").sub(/pdf$/, "_pdf")
+        name.delete("-").sub(/override$/, "_override").sub(/pdf$/, "_pdf")
           .to_sym
       end

data/lib/metanorma/version.rb CHANGED Viewed

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "2.0.5".freeze
+  VERSION = "2.0.7".freeze
 end

data/metanorma.gemspec CHANGED Viewed

@@ -24,10 +24,11 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 3.1.0"
   spec.add_runtime_dependency "asciidoctor"
+  spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "fontist", ">= 1.14.3"
   spec.add_runtime_dependency "htmlentities"
   spec.add_runtime_dependency "isodoc", ">= 2.6.3"
-  spec.add_runtime_dependency "mn2pdf", "~> 1"
+  spec.add_runtime_dependency "mn2pdf", "~> 2"
   spec.add_runtime_dependency "nokogiri"
   spec.add_runtime_dependency "shale"

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.0.7
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-30 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fontist
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -316,7 +330,9 @@ files:
 - lib/metanorma/collection/renderer/navigation.rb
 - lib/metanorma/collection/renderer/render_word.rb
 - lib/metanorma/collection/renderer/renderer.rb
+- lib/metanorma/collection/renderer/svg.rb
 - lib/metanorma/collection/renderer/utils.rb
+- lib/metanorma/collection/sectionsplit/collection.rb
 - lib/metanorma/collection/sectionsplit/sectionsplit.rb
 - lib/metanorma/collection/util/disambig_files.rb
 - lib/metanorma/collection/util/util.rb