metanorma 1.4.3 → 1.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d977a50a2b1b4c0a016439be744d32ef51fad6a4f0e5869f0ea31654065bb584
-  data.tar.gz: 3da646653cdea95e20f7e0c5ee07a9ccf301b4f73506d1f65059e03b71c064de
+  metadata.gz: b300675f26c29369f59151fc665c176cd639b324ae7b3f9f9a25d2202740d45c
+  data.tar.gz: b1ddfe077b756b1d53093c1304099fbab5df7808220e0ce20c2b5d025dc1e011
 SHA512:
-  metadata.gz: be99d0c2c11598c9e5dcd05df5da7a1bcdf49a161a56928ef8c1873bae349b401823d5f112b2aa4e0350bc20511125921c626fbd4c7ef862f6291476a409281c
-  data.tar.gz: aaba4884d254314d5f3436d6f709503a34810398f093e1010cb4186c2bb1fcd7ebc1b41725dc0542db3ce4a7896f151b826a146332d3a4a2fffb0be47f9a3482
+  metadata.gz: 988cf0f24e900c751533ecf973ba0cc73b686640bdaf5933e0078a0cfd45c6f5165f4fe7ddf438519cec42b2082e73f0b8c0d1905ff4a28d310e952ab8e68b13
+  data.tar.gz: af28119205eac475051ff399537f9b4d7a4d9f2d7c5954cac11f1360c1d0d612d3faf2a76283edac0088cf717ed0c5dfa9a0bec9da66d330375f5c9170950514

data/.github/workflows/rake.yml

@@ -9,35 +9,7 @@ on:
   pull_request:
 
 jobs:
-  rake:
-    name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
-        experimental: [ false ]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
-
-      - run: bundle exec rake
-
-  tests-passed:
-    needs: rake
-    runs-on: ubuntu-latest
-    steps:
-      - uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.METANORMA_CI_PAT_TOKEN || secrets.GITHUB_TOKEN }}
-          repository: ${{ github.repository }}
-          event-type: tests-passed
-          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'
+  notify:
+    uses: metanorma/metanorma-build-scripts/.github/workflows/inkscape-rake.yml@main
+    secrets:
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}

data/README.adoc

@@ -247,6 +247,13 @@ The options hash has the same structure it does when invoked in metanorma-cli:
 `:relaton`: exports the bibdata Relaton XML description of the document (which is part of its Metanorma XML)
 to the nominated directory
 
+== Threaded execution
+
+Metanorma has threaded execution, to generate output documents from the same Presentation XML input more quickly.
+Similar to https://github.com/relaton/relaton[relaton], the `METANORMA_PARALLEL` environment variable 
+can be used to override the default number of parallel fetches used.
+
+
 == Origin of name
 
 *Meta-* is a prefix of Greek origin ("μετα") for "`with`" "`after`".

data/lib/metanorma/collection_renderer.rb

@@ -16,7 +16,7 @@ module Metanorma
     # @param options [Hash]
     # @option options [String] :coverpage cover page HTML (Liquid template)
     # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
-    # @option options [String] :ourput_folder output directory
+    # @option options [String] :output_folder output directory
     #
     # We presuppose that the bibdata of the document is equivalent to that of
     # the collection, and that the flavour gem can sensibly process it. We may
@@ -52,7 +52,13 @@ module Metanorma
     end
 
     def dir_name_cleanse(name)
-      name.gsub(/[<>:"|?*]/, "_")
+      path = Pathname.new(name)
+      clean_regex = /[<>:"|?*]/
+      fallback_sym = "_"
+      return name.gsub(clean_regex, fallback_sym) unless path.absolute?
+
+      File.join(path.dirname,
+                path.basename.to_s.gsub(clean_regex, fallback_sym))
     end
 
     # @param col [Metanorma::Collection] XML collection

data/lib/metanorma/compile.rb

@@ -9,6 +9,7 @@ require_relative "fontist_utils"
 require_relative "util"
 require_relative "sectionsplit"
 require_relative "extract"
+require_relative "worker_pool"
 
 module Metanorma
   class Compile
@@ -33,7 +34,7 @@ module Metanorma
       extract(isodoc, options[:extract], options[:extract_type])
       FontistUtils.install_fonts(@processor, options) unless @fontist_installed
       @fontist_installed = true
-      process_extensions(filename, extensions, file, isodoc, options)
+      process_exts(filename, extensions, file, isodoc, options)
     end
 
     def require_libraries(options)
@@ -119,6 +120,16 @@ module Metanorma
       File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
     end
 
+    def relaton_export(isodoc, options)
+      return unless options[:relaton]
+
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
+      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
+      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
+      File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
+    end
+
     def export_output(fname, content, **options)
       mode = options[:binary] ? "wb" : "w:UTF-8"
       File.open(fname, mode) { |f| f.write content }
@@ -134,41 +145,68 @@ module Metanorma
     end
 
     # isodoc is Raw Metanorma XML
-    def process_extensions(filename, extensions, file, isodoc, options)
-      f = change_output_dir options
-      name = { xml: f.sub(/\.[^.]+$/, ".xml"),
-               presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+    def process_exts(filename, extensions, file, isodoc, options)
+      f = File.expand_path(change_output_dir(options))
+      fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
+                 orig_filename: File.expand_path(filename),
+                 presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+      @queue = ::Metanorma::WorkersPool
+        .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
       Util.sort_extensions_execution(extensions).each do |ext|
-        file_extension = @processor.output_formats[ext]
-        name[:out] = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        isodoc_options = get_isodoc_options(file, options, ext)
-        if ext == :rxl
-          relaton_export(isodoc, options.merge(relaton: name[:out]))
-        elsif options[:passthrough_presentation_xml] && ext == :presentation
-          FileUtils.cp filename, name[:presentationxml]
-        elsif ext == :html && options[:sectionsplit]
-          sectionsplit_convert(name[:xml], isodoc, name[:out], isodoc_options)
-        else
-          if ext == :pdf && FontistUtils.has_fonts_manifest?(@processor,
-                                                             options)
-            isodoc_options[:mn2pdf] = {
-              font_manifest: FontistUtils.location_manifest(@processor),
-            }
-          end
-          process_extensions1(ext, name, isodoc, isodoc_options)
-        end
-        wrap_html(options, file_extension, name[:out])
+        process_ext(ext, file, isodoc, fnames, options)
+      end
+      @queue.shutdown
+    end
+
+    def process_ext(ext, file, isodoc, fnames, options)
+      fnames[:ext] = @processor.output_formats[ext]
+      fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
+      isodoc_options = get_isodoc_options(file, options, ext)
+      thread = nil
+      unless process_ext_simple(ext, isodoc, fnames, options,
+                                isodoc_options)
+        thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
+      end
+      thread
+    end
+
+    def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
+      if ext == :rxl
+        relaton_export(isodoc, options.merge(relaton: fnames[:out]))
+      elsif options[:passthrough_presentation_xml] && ext == :presentation
+        f = File.exists?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
+        FileUtils.cp f, fnames[:presentationxml]
+      elsif ext == :html && options[:sectionsplit]
+        sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
+                             isodoc_options)
+      else return false
       end
+      true
     end
 
-    def process_extensions1(ext, fnames, isodoc, isodoc_options)
+    def process_exts1(ext, fnames, isodoc, options, isodoc_options)
       if @processor.use_presentation_xml(ext)
-        @processor.output(nil, fnames[:presentationxml], fnames[:out], ext,
-                          isodoc_options)
+        @queue.schedule(ext, fnames.dup, options.dup,
+                        isodoc_options.dup) do |a, b, c, d|
+          process_output_threaded(a, b, c, d)
+        end
       else
-        @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
-                          isodoc_options)
+        process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
       end
+    end
+
+    def process_output_threaded(ext, fnames1, options1, isodoc_options1)
+      @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
+                        isodoc_options1)
+      wrap_html(options1, fnames1[:ext], fnames1[:out])
+    rescue StandardError => e
+      isodoc_error_process(e)
+    end
+
+    def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
+      @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
+                        isodoc_options)
+      nil # return as Thread
     rescue StandardError => e
       isodoc_error_process(e)
     end
@@ -184,15 +222,17 @@ module Metanorma
       end
     end
 
-    def get_isodoc_options(file, options, _ext)
-      isodoc_options = @processor.extract_options(file)
-      isodoc_options[:datauriimage] = true if options[:datauriimage]
-      isodoc_options[:sourcefilename] = options[:filename]
-      %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements)
-        .each do |x|
-        isodoc_options[x] ||= options[x]
-      end
-      isodoc_options
+    def get_isodoc_options(file, options, ext)
+      ret = @processor.extract_options(file)
+      ret[:datauriimage] = true if options[:datauriimage]
+      ret[:sourcefilename] = options[:filename]
+      %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements
+         tocfigures toctables tocrecommendations)
+        .each { |x| ret[x] ||= options[x] }
+      ext == :pdf && FontistUtils.has_fonts_manifest?(@processor, options) and
+        ret[:mn2pdf] =
+          { font_manifest: FontistUtils.location_manifest(@processor) }
+      ret
     end
 
     # @param options [Hash]

data/lib/metanorma/input/asciidoc.rb

@@ -68,6 +68,7 @@ module Metanorma
            pdf-owner-password pdf-allow-copy-content pdf-allow-edit-content
            pdf-allow-assemble-document pdf-allow-edit-annotations
            pdf-allow-print pdf-allow-print-hq pdf-allow-fill-in-forms
+           toc-figures toc-tables toc-recommendations
            pdf-allow-access-content pdf-encrypt-metadata).freeze
 
       def extract_options(file)

data/lib/metanorma/registry.rb

@@ -1,7 +1,7 @@
 # Registry of all Metanorma types and entry points
 #
 
-require 'singleton'
+require "singleton"
 
 class Error < StandardError
 end
@@ -14,15 +14,16 @@ module Metanorma
 
     def initialize
       @processors = {}
-      @aliases = {csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa}
+      @aliases = { csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa }
     end
 
     def alias(x)
       @aliases[x]
     end
 
-    def register processor
+    def register(processor)
       raise Error unless processor < ::Metanorma::Processor
+
       p = processor.new
       # p.short[-1] is the canonical name
       short = Array(p.short)
@@ -31,7 +32,8 @@ module Metanorma
         @aliases[s] = short[-1]
       end
       Array(p.short)
-      Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered", :info)
+      Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered",
+               :info)
     end
 
     def find_processor(short)
@@ -42,21 +44,17 @@ module Metanorma
       @processors.keys
     end
 
-    def processors
-      @processors
-    end
-
     def output_formats
-      @processors.inject({}) do |acc, (k,v)|
+      @processors.inject({}) do |acc, (k, v)|
         acc[k] = v.output_formats
         acc
       end
     end
 
     def root_tags
-      @processors.inject({}) do |acc, (k,v)|
+      @processors.inject({}) do |acc, (k, v)|
         if v.asciidoctor_backend
-          x = Asciidoctor.load nil, {backend: v.asciidoctor_backend}
+          x = Asciidoctor.load nil, { backend: v.asciidoctor_backend }
           acc[k] = x.converter.xml_root_tag
         end
         acc

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.4.3".freeze
+  VERSION = "1.4.6".freeze
 end

data/lib/metanorma/worker_pool.rb

@@ -0,0 +1,29 @@
+module Metanorma
+  class WorkersPool
+    def initialize(workers)
+      @workers = workers
+      @queue = SizedQueue.new(@workers)
+      @threads = Array.new(@workers) do
+        Thread.new do
+          catch(:exit) do
+            loop do
+              job, args = @queue.pop
+              job.call *args
+            end
+          end
+        end
+      end
+    end
+
+    def schedule(*args, &block)
+      @queue << [block, args]
+    end
+
+    def shutdown
+      @workers.times do
+        schedule { throw :exit }
+      end
+      @threads.map(&:join)
+    end
+  end
+end

data/metanorma.gemspec

@@ -36,12 +36,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "metanorma-iso", "~> 1.10"
+  spec.add_development_dependency "metanorma-iso", "~> 2.0"
   spec.add_development_dependency "mnconvert"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-command", "~> 1.0"
   spec.add_development_dependency "rubocop", "~> 1.5.2"
   spec.add_development_dependency "sassc", "~> 2.4.0"
-  spec.add_development_dependency "reline", "~> 0.2.8.pre.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.4.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-13 00:00:00.000000000 Z
+date: 2022-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: mnconvert
   requirement: !ruby/object:Gem::Requirement
@@ -234,20 +234,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.4.0
-- !ruby/object:Gem::Dependency
-  name: reline
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.2.8.pre.11
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.2.8.pre.11
 description: Library to process any Metanorma standard.
 email:
 - open.source@ribose.com
@@ -298,6 +284,7 @@ files:
 - lib/metanorma/sectionsplit.rb
 - lib/metanorma/util.rb
 - lib/metanorma/version.rb
+- lib/metanorma/worker_pool.rb
 - metanorma.gemspec
 homepage: https://github.com/metanorma/metanorma
 licenses: