metanorma 1.4.3 → 1.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d977a50a2b1b4c0a016439be744d32ef51fad6a4f0e5869f0ea31654065bb584
-  data.tar.gz: 3da646653cdea95e20f7e0c5ee07a9ccf301b4f73506d1f65059e03b71c064de
+  metadata.gz: c8457479b196c4d3db250ce33f650cd822f1ac8f1a510d1b6ac9653038446e1e
+  data.tar.gz: f1219e134ab2113cd3e25c5bca39fe3194a3a038f9b6500f0d5e35d66f168d0d
 SHA512:
-  metadata.gz: be99d0c2c11598c9e5dcd05df5da7a1bcdf49a161a56928ef8c1873bae349b401823d5f112b2aa4e0350bc20511125921c626fbd4c7ef862f6291476a409281c
-  data.tar.gz: aaba4884d254314d5f3436d6f709503a34810398f093e1010cb4186c2bb1fcd7ebc1b41725dc0542db3ce4a7896f151b826a146332d3a4a2fffb0be47f9a3482
+  metadata.gz: 8e62d329f40ca168b5f2250ae86e676b89650c754233a4d249eea45bdf7652863ce5824a128bbff0c6225435a97be4bb8b420ce9cfca44587be9d1d852c33599
+  data.tar.gz: 1b18636330ca30bba9f2a0080e1728c6d05763349ffafebab78e1b61043f3b45c5160156dfdbf541d886fc14e195ee54808cbfbc4dd8d296e7bb56ee5edca735

data/README.adoc

@@ -247,6 +247,13 @@ The options hash has the same structure it does when invoked in metanorma-cli:
 `:relaton`: exports the bibdata Relaton XML description of the document (which is part of its Metanorma XML)
 to the nominated directory
 
+== Threaded execution
+
+Metanorma has threaded execution, to generate output documents from the same Presentation XML input more quickly.
+Similar to https://github.com/relaton/relaton[relaton], the `METANORMA_PARALLEL` environment variable 
+can be used to override the default number of parallel fetches used.
+
+
 == Origin of name
 
 *Meta-* is a prefix of Greek origin ("μετα") for "`with`" "`after`".

data/lib/metanorma/collection_renderer.rb

@@ -16,7 +16,7 @@ module Metanorma
     # @param options [Hash]
     # @option options [String] :coverpage cover page HTML (Liquid template)
     # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
-    # @option options [String] :ourput_folder output directory
+    # @option options [String] :output_folder output directory
     #
     # We presuppose that the bibdata of the document is equivalent to that of
     # the collection, and that the flavour gem can sensibly process it. We may

data/lib/metanorma/compile.rb

@@ -9,6 +9,7 @@ require_relative "fontist_utils"
 require_relative "util"
 require_relative "sectionsplit"
 require_relative "extract"
+require_relative "worker_pool"
 
 module Metanorma
   class Compile
@@ -33,7 +34,7 @@ module Metanorma
       extract(isodoc, options[:extract], options[:extract_type])
       FontistUtils.install_fonts(@processor, options) unless @fontist_installed
       @fontist_installed = true
-      process_extensions(filename, extensions, file, isodoc, options)
+      process_exts(filename, extensions, file, isodoc, options)
     end
 
     def require_libraries(options)
@@ -119,6 +120,16 @@ module Metanorma
       File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
     end
 
+    def relaton_export(isodoc, options)
+      return unless options[:relaton]
+
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
+      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
+      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
+      File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
+    end
+
     def export_output(fname, content, **options)
       mode = options[:binary] ? "wb" : "w:UTF-8"
       File.open(fname, mode) { |f| f.write content }
@@ -134,41 +145,68 @@ module Metanorma
     end
 
     # isodoc is Raw Metanorma XML
-    def process_extensions(filename, extensions, file, isodoc, options)
-      f = change_output_dir options
-      name = { xml: f.sub(/\.[^.]+$/, ".xml"),
-               presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+    def process_exts(filename, extensions, file, isodoc, options)
+      f = File.expand_path(change_output_dir(options))
+      fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
+                 orig_filename: File.expand_path(filename),
+                 presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+      @queue = ::Metanorma::WorkersPool
+        .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
       Util.sort_extensions_execution(extensions).each do |ext|
-        file_extension = @processor.output_formats[ext]
-        name[:out] = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        isodoc_options = get_isodoc_options(file, options, ext)
-        if ext == :rxl
-          relaton_export(isodoc, options.merge(relaton: name[:out]))
-        elsif options[:passthrough_presentation_xml] && ext == :presentation
-          FileUtils.cp filename, name[:presentationxml]
-        elsif ext == :html && options[:sectionsplit]
-          sectionsplit_convert(name[:xml], isodoc, name[:out], isodoc_options)
-        else
-          if ext == :pdf && FontistUtils.has_fonts_manifest?(@processor,
-                                                             options)
-            isodoc_options[:mn2pdf] = {
-              font_manifest: FontistUtils.location_manifest(@processor),
-            }
-          end
-          process_extensions1(ext, name, isodoc, isodoc_options)
-        end
-        wrap_html(options, file_extension, name[:out])
+        process_ext(ext, file, isodoc, fnames, options)
+      end
+      @queue.shutdown
+    end
+
+    def process_ext(ext, file, isodoc, fnames, options)
+      fnames[:ext] = @processor.output_formats[ext]
+      fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
+      isodoc_options = get_isodoc_options(file, options, ext)
+      thread = nil
+      unless process_ext_simple(ext, isodoc, fnames, options,
+                                isodoc_options)
+        thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
+      end
+      thread
+    end
+
+    def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
+      if ext == :rxl
+        relaton_export(isodoc, options.merge(relaton: fnames[:out]))
+      elsif options[:passthrough_presentation_xml] && ext == :presentation
+        f = File.exists?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
+        FileUtils.cp f, fnames[:presentationxml]
+      elsif ext == :html && options[:sectionsplit]
+        sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
+                             isodoc_options)
+      else return false
       end
+      true
     end
 
-    def process_extensions1(ext, fnames, isodoc, isodoc_options)
+    def process_exts1(ext, fnames, isodoc, options, isodoc_options)
       if @processor.use_presentation_xml(ext)
-        @processor.output(nil, fnames[:presentationxml], fnames[:out], ext,
-                          isodoc_options)
+        @queue.schedule(ext, fnames.dup, options.dup,
+                        isodoc_options.dup) do |a, b, c, d|
+          process_output_threaded(a, b, c, d)
+        end
       else
-        @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
-                          isodoc_options)
+        process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
       end
+    end
+
+    def process_output_threaded(ext, fnames1, options1, isodoc_options1)
+      @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
+                        isodoc_options1)
+      wrap_html(options1, fnames1[:ext], fnames1[:out])
+    rescue StandardError => e
+      isodoc_error_process(e)
+    end
+
+    def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
+      @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
+                        isodoc_options)
+      nil # return as Thread
     rescue StandardError => e
       isodoc_error_process(e)
     end
@@ -184,15 +222,16 @@ module Metanorma
       end
     end
 
-    def get_isodoc_options(file, options, _ext)
-      isodoc_options = @processor.extract_options(file)
-      isodoc_options[:datauriimage] = true if options[:datauriimage]
-      isodoc_options[:sourcefilename] = options[:filename]
+    def get_isodoc_options(file, options, ext)
+      ret = @processor.extract_options(file)
+      ret[:datauriimage] = true if options[:datauriimage]
+      ret[:sourcefilename] = options[:filename]
       %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements)
-        .each do |x|
-        isodoc_options[x] ||= options[x]
-      end
-      isodoc_options
+        .each { |x| ret[x] ||= options[x] }
+      ext == :pdf && FontistUtils.has_fonts_manifest?(@processor, options) and
+        ret[:mn2pdf] =
+          { font_manifest: FontistUtils.location_manifest(@processor) }
+      ret
     end
 
     # @param options [Hash]

data/lib/metanorma/registry.rb

@@ -1,7 +1,7 @@
 # Registry of all Metanorma types and entry points
 #
 
-require 'singleton'
+require "singleton"
 
 class Error < StandardError
 end
@@ -14,15 +14,16 @@ module Metanorma
 
     def initialize
       @processors = {}
-      @aliases = {csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa}
+      @aliases = { csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa }
     end
 
     def alias(x)
       @aliases[x]
     end
 
-    def register processor
+    def register(processor)
       raise Error unless processor < ::Metanorma::Processor
+
       p = processor.new
       # p.short[-1] is the canonical name
       short = Array(p.short)
@@ -31,7 +32,8 @@ module Metanorma
         @aliases[s] = short[-1]
       end
       Array(p.short)
-      Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered", :info)
+      Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered",
+               :info)
     end
 
     def find_processor(short)
@@ -42,21 +44,17 @@ module Metanorma
       @processors.keys
     end
 
-    def processors
-      @processors
-    end
-
     def output_formats
-      @processors.inject({}) do |acc, (k,v)|
+      @processors.inject({}) do |acc, (k, v)|
         acc[k] = v.output_formats
         acc
       end
     end
 
     def root_tags
-      @processors.inject({}) do |acc, (k,v)|
+      @processors.inject({}) do |acc, (k, v)|
         if v.asciidoctor_backend
-          x = Asciidoctor.load nil, {backend: v.asciidoctor_backend}
+          x = Asciidoctor.load nil, { backend: v.asciidoctor_backend }
           acc[k] = x.converter.xml_root_tag
         end
         acc

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.4.3".freeze
+  VERSION = "1.4.4".freeze
 end

data/lib/metanorma/worker_pool.rb

@@ -0,0 +1,29 @@
+module Metanorma
+  class WorkersPool
+    def initialize(workers)
+      @workers = workers
+      @queue = SizedQueue.new(@workers)
+      @threads = Array.new(@workers) do
+        Thread.new do
+          catch(:exit) do
+            loop do
+              job, args = @queue.pop
+              job.call *args
+            end
+          end
+        end
+      end
+    end
+
+    def schedule(*args, &block)
+      @queue << [block, args]
+    end
+
+    def shutdown
+      @workers.times do
+        schedule { throw :exit }
+      end
+      @threads.map(&:join)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.4.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-13 00:00:00.000000000 Z
+date: 2021-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -298,6 +298,7 @@ files:
 - lib/metanorma/sectionsplit.rb
 - lib/metanorma/util.rb
 - lib/metanorma/version.rb
+- lib/metanorma/worker_pool.rb
 - metanorma.gemspec
 homepage: https://github.com/metanorma/metanorma
 licenses: