metanorma 1.4.0 → 1.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cebdc5e3a04db33d1bf3c0042d914f03a1e7c36ae67d625138fb1499679abdcf
-  data.tar.gz: 82f7a4b816cea046181cfd05da94ca21a52f1c12ac02d9f6f2e89ae1b07e6f60
+  metadata.gz: c8457479b196c4d3db250ce33f650cd822f1ac8f1a510d1b6ac9653038446e1e
+  data.tar.gz: f1219e134ab2113cd3e25c5bca39fe3194a3a038f9b6500f0d5e35d66f168d0d
 SHA512:
-  metadata.gz: cfa8565f3b0a55f12f0d0e8971d82c8bf619390cfc76860514dda7b1801267d7e04719161048ff7d9b16a71e238df2e24e145198229b5b5930233ae9110933ec
-  data.tar.gz: 81764ee18d5c1b36f3aacf19c0a4cf4b02cd4fb0a8979fe9d0a6733a93f4f05ddc845be35b25ec38be31d595fd04a019ae6aaf6ca33a97664519550e1abe280b
+  metadata.gz: 8e62d329f40ca168b5f2250ae86e676b89650c754233a4d249eea45bdf7652863ce5824a128bbff0c6225435a97be4bb8b420ce9cfca44587be9d1d852c33599
+  data.tar.gz: 1b18636330ca30bba9f2a0080e1728c6d05763349ffafebab78e1b61043f3b45c5160156dfdbf541d886fc14e195ee54808cbfbc4dd8d296e7bb56ee5edca735

data/Gemfile

@@ -9,3 +9,4 @@ gemspec
 if File.exist? "Gemfile.devel"
   eval File.read("Gemfile.devel"), nil, "Gemfile.devel" # rubocop:disable Security/Eval
 end
+

data/README.adoc

@@ -247,6 +247,13 @@ The options hash has the same structure it does when invoked in metanorma-cli:
 `:relaton`: exports the bibdata Relaton XML description of the document (which is part of its Metanorma XML)
 to the nominated directory
 
+== Threaded execution
+
+Metanorma has threaded execution, to generate output documents from the same Presentation XML input more quickly.
+Similar to https://github.com/relaton/relaton[relaton], the `METANORMA_PARALLEL` environment variable 
+can be used to override the default number of parallel fetches used.
+
+
 == Origin of name
 
 *Meta-* is a prefix of Greek origin ("μετα") for "`with`" "`after`".

data/lib/metanorma/collection.rb

@@ -91,12 +91,9 @@ module Metanorma
       private
 
       def parse_xml(file)
-        xml = Nokogiri::XML File.read(file, encoding: "UTF-8") do |config|
-          config.huge
-        end
-        if (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata"))
-          bd = Relaton::Cli.parse_xml b
-        end
+        xml = Nokogiri::XML File.read(file, encoding: "UTF-8") { |c| c.huge }
+        (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata")) and
+          bd = Relaton::Cli.parse_xml(b)
         mnf_xml = xml.at("/xmlns:metanorma-collection/xmlns:manifest")
         mnf = CollectionManifest.from_xml mnf_xml
         pref = pref_final_content xml.at("//xmlns:prefatory-content")

data/lib/metanorma/collection_fileparse.rb

@@ -83,11 +83,15 @@ module Metanorma
       add_document_suffix(identifier, docxml)
       update_direct_refs_to_docs(docxml, identifier)
       svgmap_resolve(datauri_encode(docxml))
+      hide_refs(docxml)
+      docxml.to_xml
+    end
+
+    def hide_refs(docxml)
       docxml.xpath(ns("//references[bibitem][not(./bibitem[not(@hidden) or "\
                       "@hidden = 'false'])]")).each do |f|
         f["hidden"] = "true"
       end
-      docxml.to_xml
     end
 
     def supply_repo_ids(docxml)
@@ -97,7 +101,8 @@ module Metanorma
         b.xpath(ns("./docidentifier")).each do |d|
           next unless @files[d.text]
 
-          d.next = "<docidentifier type='repository'>current-metanorma-collection/#{d.text}"
+          d.next = "<docidentifier type='repository'>"\
+                   "current-metanorma-collection/#{d.text}"
         end
       end
     end
@@ -111,29 +116,30 @@ module Metanorma
 
     def svgmap_resolve(docxml)
       isodoc = IsoDoc::Convert.new({})
+      isodoc.bibitem_lookup(docxml)
       docxml.xpath(ns("//svgmap//eref")).each do |e|
-        href = isodoc.eref_target(e)
-        next if href == "##{e['bibitemid']}" ||
-          href =~ /^#/ && !docxml.at("//*[@id = '#{href.sub(/^#/, '')}']")
-
-        e["target"] = href.strip
-        e.name = "link"
-        e&.elements&.remove
+        svgmap_resolve1(e, isodoc)
       end
       Metanorma::Utils::svgmap_rewrite(docxml, "")
     end
 
+    def svgmap_resolve1(eref, isodoc)
+      href = isodoc.eref_target(eref)
+      return if href == "##{eref['bibitemid']}" ||
+        (href =~ /^#/ && !docxml.at("//*[@id = '#{href.sub(/^#/, '')}']"))
+
+      eref["target"] = href.strip
+      eref.name = "link"
+      eref&.elements&.remove
+    end
+
     # repo(current-metanorma-collection/ISO 17301-1:2016)
     # replaced by bibdata of "ISO 17301-1:2016" in situ as bibitem.
     # Any erefs to that bibitem id are replaced with relative URL
     # Preferably with anchor, and is a job to realise dynamic lookup
     # of localities.
     def update_direct_refs_to_docs(docxml, identifier)
-      erefs = docxml.xpath(ns("//eref"))
-        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
-        m[:citeas][i["citeas"]] = true
-        m[:bibitemid][i["bibitemid"]] = true
-      end
+      erefs = collect_erefs(docxml)
       docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
         docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
         next unless docid && %r{^current-metanorma-collection/}.match(docid)
@@ -144,6 +150,14 @@ module Metanorma
       end
     end
 
+    def collect_erefs(docxml)
+      docxml.xpath(ns("//eref"))
+        .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
+        m[:citeas][i["citeas"]] = true
+        m[:bibitemid][i["bibitemid"]] = true
+      end
+    end
+
     # Resolve erefs to a container of ids in another doc,
     # to an anchor eref (direct link)
     def update_indirect_refs_to_docs(docxml, internal_refs)

data/lib/metanorma/collection_fileprocess.rb

@@ -99,17 +99,15 @@ module Metanorma
       out = ref["attachment"] ? ref["fileref"] : File.basename(ref["fileref"])
       ret = if ref["fileref"]
               { type: "fileref", ref: @documents[identifier].file,
-                rel_path: ref["fileref"],
-                out_path: out }
-            else
-              { type: "id", ref: ref["id"] }
+                rel_path: ref["fileref"], out_path: out }
+            else { type: "id", ref: ref["id"] }
             end
       %i(attachment sectionsplit index).each do |s|
         ret[s] = ref[s.to_s] if ref[s.to_s]
       end
-      ret[:presentationxml] = ref["presentation-xml"] if ref["presentation-xml"]
-      ret[:bareafterfirst] = ref["bare-after-first"] if ref["bare-after-first"]
-      ret
+      ret[:presentationxml] = ref["presentation-xml"]
+      ret[:bareafterfirst] = ref["bare-after-first"]
+      ret.compact
     end
 
     def add_suffix_to_attributes(doc, suffix, tag_name, attribute_name)
@@ -164,10 +162,15 @@ module Metanorma
     def file_compile(file, filename, identifier)
       return if @files[identifier][:sectionsplit] == "true"
 
-      @compile.compile file.path, { format: :asciidoc, extension_keys: @format }
-        .merge(compile_options(identifier))
+      opts = {
+        format: :asciidoc,
+        extension_keys: @format,
+        output_dir: @outdir,
+      }.merge(compile_options(identifier))
+
+      @compile.compile file, opts
       @files[identifier][:outputs] = {}
-      file_compile_formats(file, filename, identifier)
+      file_compile_formats(filename, identifier)
     end
 
     def compile_options(identifier)
@@ -182,29 +185,18 @@ module Metanorma
       ret
     end
 
-    def file_compile_formats(file, filename, identifier)
+    def file_compile_formats(filename, identifier)
+      file_id = @files[identifier]
+      @format << :presentation if @format.include?(:pdf)
       @format.each do |e|
         ext = @compile.processor.output_formats[e]
         fn = File.basename(filename).sub(/(?<=\.)[^.]+$/, ext.to_s)
-        if /html$/.match?(ext) && @files[identifier][:sectionsplit]
-          # file_sectionsplit_copy(file, fn, identifier, ext, e)
-        else
-          FileUtils.cp file.path.sub(/\.xml$/, ".#{ext}"),
-                       File.join(@outdir, fn)
-          @files[identifier][:outputs][e] = File.join(@outdir, fn)
+        unless /html$/.match?(ext) && file_id[:sectionsplit]
+          file_id[:outputs][e] = File.join(@outdir, fn)
         end
       end
     end
 
-    def file_sectionsplit_copy(file, base, identifier, ext, format)
-      dir = file.path.sub(/\.xml$/, ".#{ext}_collection")
-      files = Dir.glob("#{dir}/*.#{ext}")
-      FileUtils.cp files, @outdir
-      cover = File.join(@outdir, base.sub(/\.html$/, ".index.html"))
-      FileUtils.cp File.join(dir, "index.html"), cover
-      @files[identifier][:outputs][format] = cover
-    end
-
     def copy_file_to_dest(fileref)
       dest = File.join(@outdir, fileref[:out_path])
       FileUtils.mkdir_p(File.dirname(dest))
@@ -223,12 +215,13 @@ module Metanorma
         else
           file, filename = targetfile(x, read: true)
           warn "\n\n\n\n\nProcess #{filename}: #{DateTime.now.strftime('%H:%M:%S')}"
-          file = update_xrefs(file, identifier, internal_refs)
-          Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
-            f.write(file)
-            f.close
-            file_compile(f, filename, identifier)
-          end
+          collection_xml = update_xrefs(file, identifier, internal_refs)
+          collection_filename = File.basename(filename, File.extname(filename))
+          collection_xml_path = File.join(Dir.tmpdir,
+                                          "#{collection_filename}.xml")
+          File.write collection_xml_path, collection_xml, encoding: "UTF-8"
+          file_compile(collection_xml_path, filename, identifier)
+          FileUtils.rm(collection_xml_path)
         end
       end
     end

data/lib/metanorma/collection_renderer.rb

@@ -16,7 +16,7 @@ module Metanorma
     # @param options [Hash]
     # @option options [String] :coverpage cover page HTML (Liquid template)
     # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
-    # @option options [String] :ourput_folder output directory
+    # @option options [String] :output_folder output directory
     #
     # We presuppose that the bibdata of the document is equivalent to that of
     # the collection, and that the flavour gem can sensibly process it. We may
@@ -62,7 +62,6 @@ module Metanorma
     # @option options [Strong] :ourput_folder output directory
     def self.render(col, options = {})
       folder = File.dirname col.file
-      # require "byebug"; byebug
       warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
       cr = new(col, folder, options)
       warn "\n\n\n\n\nRender Files: #{DateTime.now.strftime('%H:%M:%S')}"

data/lib/metanorma/compile.rb

@@ -8,6 +8,8 @@ require_relative "compile_validate"
 require_relative "fontist_utils"
 require_relative "util"
 require_relative "sectionsplit"
+require_relative "extract"
+require_relative "worker_pool"
 
 module Metanorma
   class Compile
@@ -32,7 +34,7 @@ module Metanorma
       extract(isodoc, options[:extract], options[:extract_type])
       FontistUtils.install_fonts(@processor, options) unless @fontist_installed
       @fontist_installed = true
-      process_extensions(extensions, file, isodoc, options)
+      process_exts(filename, extensions, file, isodoc, options)
     end
 
     def require_libraries(options)
@@ -128,112 +130,85 @@ module Metanorma
       File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
     end
 
-    def clean_sourcecode(xml)
-      xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
-                ".//xmlns:annotation").each(&:remove)
-      xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
-      HTMLEntities.new.decode(xml.children.to_xml)
+    def export_output(fname, content, **options)
+      mode = options[:binary] ? "wb" : "w:UTF-8"
+      File.open(fname, mode) { |f| f.write content }
     end
 
-    def extract(isodoc, dirname, extract_types)
-      return unless dirname
-
-      if extract_types.nil? || extract_types.empty?
-        extract_types = %i[sourcecode image requirement]
+    def wrap_html(options, file_extension, outfilename)
+      if options[:wrapper] && /html$/.match(file_extension)
+        outfilename = outfilename.sub(/\.html$/, "")
+        FileUtils.mkdir_p outfilename
+        FileUtils.mv "#{outfilename}.html", outfilename
+        FileUtils.mv "#{outfilename}_images", outfilename, force: true
       end
-      FileUtils.rm_rf dirname
-      FileUtils.mkdir_p dirname
-      xml = Nokogiri::XML(isodoc) { |config| config.huge }
-      sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
-      image_export(xml, dirname) if extract_types.include? :image
-      requirement_export(xml, dirname) if extract_types.include? :requirement
     end
 
-    def sourcecode_export(xml, dirname)
-      xml.at("//sourcecode | //xmlns:sourcecode") or return
-      FileUtils.mkdir_p "#{dirname}/sourcecode"
-      xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
-        filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
-        File.open("#{dirname}/sourcecode/#{filename}", "w:UTF-8") do |f|
-          f.write clean_sourcecode(s.dup)
-        end
+    # isodoc is Raw Metanorma XML
+    def process_exts(filename, extensions, file, isodoc, options)
+      f = File.expand_path(change_output_dir(options))
+      fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
+                 orig_filename: File.expand_path(filename),
+                 presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+      @queue = ::Metanorma::WorkersPool
+        .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
+      Util.sort_extensions_execution(extensions).each do |ext|
+        process_ext(ext, file, isodoc, fnames, options)
       end
+      @queue.shutdown
     end
 
-    def image_export(xml, dirname)
-      xml.at("//image | //xmlns:image") or return
-      FileUtils.mkdir_p "#{dirname}/image"
-      xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
-        next unless /^data:image/.match? s["src"]
-
-        %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
-        filename = s["filename"] || sprintf("image-%04d.%s", i, imgtype)
-        File.open("#{dirname}/image/#{filename}", "wb") do |f|
-          f.write(Base64.strict_decode64(imgdata))
-        end
+    def process_ext(ext, file, isodoc, fnames, options)
+      fnames[:ext] = @processor.output_formats[ext]
+      fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
+      isodoc_options = get_isodoc_options(file, options, ext)
+      thread = nil
+      unless process_ext_simple(ext, isodoc, fnames, options,
+                                isodoc_options)
+        thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
       end
+      thread
+    end
+
+    def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
+      if ext == :rxl
+        relaton_export(isodoc, options.merge(relaton: fnames[:out]))
+      elsif options[:passthrough_presentation_xml] && ext == :presentation
+        f = File.exists?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
+        FileUtils.cp f, fnames[:presentationxml]
+      elsif ext == :html && options[:sectionsplit]
+        sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
+                             isodoc_options)
+      else return false
+      end
+      true
     end
 
-    REQUIREMENT_XPATH = "//requirement | //xmlns:requirement | "\
-                        "//recommendation | //xmlns:recommendation | //permission | "\
-                        "//xmlns:permission".freeze
-
-    def requirement_export(xml, dirname)
-      xml.at(REQUIREMENT_XPATH) or return
-      FileUtils.mkdir_p "#{dirname}/requirement"
-      xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
-        filename = s["filename"] || sprintf("%s-%04d.xml", s.name, i)
-        File.open("#{dirname}/requirement/#{filename}", "w:UTF-8") do |f|
-          f.write s
+    def process_exts1(ext, fnames, isodoc, options, isodoc_options)
+      if @processor.use_presentation_xml(ext)
+        @queue.schedule(ext, fnames.dup, options.dup,
+                        isodoc_options.dup) do |a, b, c, d|
+          process_output_threaded(a, b, c, d)
         end
+      else
+        process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
       end
     end
 
-    def wrap_html(options, file_extension, outfilename)
-      if options[:wrapper] && /html$/.match(file_extension)
-        outfilename = outfilename.sub(/\.html$/, "")
-        FileUtils.mkdir_p outfilename
-        FileUtils.mv "#{outfilename}.html", outfilename
-        FileUtils.mv "#{outfilename}_images", outfilename, force: true
-      end
+    def process_output_threaded(ext, fnames1, options1, isodoc_options1)
+      @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
+                        isodoc_options1)
+      wrap_html(options1, fnames1[:ext], fnames1[:out])
+    rescue StandardError => e
+      isodoc_error_process(e)
     end
 
-    # isodoc is Raw Metanorma XML
-    def process_extensions(extensions, file, isodoc, options)
-      f = change_output_dir options
-      xml_name = f.sub(/\.[^.]+$/, ".xml")
-      presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
-      Util.sort_extensions_execution(extensions).each do |ext|
-        file_extension = @processor.output_formats[ext]
-        outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
-        isodoc_options = get_isodoc_options(file, options, ext)
-        if ext == :rxl
-          relaton_export(isodoc, options.merge(relaton: outfilename))
-        elsif options[:passthrough_presentation_xml] && ext == :presentation
-          FileUtils.cp f, presentationxml_name
-        elsif ext == :html && options[:sectionsplit]
-          sectionsplit_convert(xml_name, isodoc, outfilename, isodoc_options)
-        else
-          if ext == :pdf && FontistUtils.has_fonts_manifest?(@processor,
-                                                             options)
-            isodoc_options[:mn2pdf] = {
-              font_manifest: FontistUtils.location_manifest(@processor),
-            }
-          end
-          begin
-            if @processor.use_presentation_xml(ext)
-              @processor.output(nil, presentationxml_name, outfilename, ext,
-                                isodoc_options)
-            else
-              @processor.output(isodoc, xml_name, outfilename, ext,
-                                isodoc_options)
-            end
-          rescue StandardError => e
-            isodoc_error_process(e)
-          end
-        end
-        wrap_html(options, file_extension, outfilename)
-      end
+    def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
+      @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
+                        isodoc_options)
+      nil # return as Thread
+    rescue StandardError => e
+      isodoc_error_process(e)
     end
 
     private
@@ -248,14 +223,15 @@ module Metanorma
     end
 
     def get_isodoc_options(file, options, ext)
-      isodoc_options = @processor.extract_options(file)
-      isodoc_options[:datauriimage] = true if options[:datauriimage]
-      isodoc_options[:sourcefilename] = options[:filename]
+      ret = @processor.extract_options(file)
+      ret[:datauriimage] = true if options[:datauriimage]
+      ret[:sourcefilename] = options[:filename]
       %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements)
-        .each do |x|
-        isodoc_options[x] ||= options[x]
-      end
-      isodoc_options
+        .each { |x| ret[x] ||= options[x] }
+      ext == :pdf && FontistUtils.has_fonts_manifest?(@processor, options) and
+        ret[:mn2pdf] =
+          { font_manifest: FontistUtils.location_manifest(@processor) }
+      ret
     end
 
     # @param options [Hash]

data/lib/metanorma/extract.rb

@@ -0,0 +1,72 @@
+module Metanorma
+  class Compile
+    def relaton_export(isodoc, options)
+      return unless options[:relaton]
+
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
+      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
+      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
+      File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
+    end
+
+    def clean_sourcecode(xml)
+      xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
+                ".//xmlns:annotation").each(&:remove)
+      xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
+      HTMLEntities.new.decode(xml.children.to_xml)
+    end
+
+    def extract(isodoc, dirname, extract_types)
+      return unless dirname
+
+      extract_types.nil? || extract_types.empty? and
+        extract_types = %i[sourcecode image requirement]
+      FileUtils.rm_rf dirname
+      FileUtils.mkdir_p dirname
+      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
+      image_export(xml, dirname) if extract_types.include? :image
+      extract_types.include?(:requirement) and
+        requirement_export(xml, dirname)
+    end
+
+    def sourcecode_export(xml, dirname)
+      xml.at("//sourcecode | //xmlns:sourcecode") or return
+      FileUtils.mkdir_p "#{dirname}/sourcecode"
+      xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
+        filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
+        export_output("#{dirname}/sourcecode/#{filename}",
+                      clean_sourcecode(s.dup))
+      end
+    end
+
+    def image_export(xml, dirname)
+      xml.at("//image | //xmlns:image") or return
+      FileUtils.mkdir_p "#{dirname}/image"
+      xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
+        next unless /^data:image/.match? s["src"]
+
+        %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
+        fn = s["filename"] || sprintf("image-%<num>04d.%<name>s",
+                                      num: i, name: imgtype)
+        export_output("#{dirname}/image/#{fn}", Base64.strict_decode64(imgdata),
+                      binary: true)
+      end
+    end
+
+    REQUIREMENT_XPATH =
+      "//requirement | //xmlns:requirement | //recommendation | "\
+      "//xmlns:recommendation | //permission | //xmlns:permission".freeze
+
+    def requirement_export(xml, dirname)
+      xml.at(REQUIREMENT_XPATH) or return
+      FileUtils.mkdir_p "#{dirname}/requirement"
+      xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
+        fn = s["filename"] ||
+          sprintf("%<name>s-%<num>04d.xml", name: s.name, num: i)
+        export_output("#{dirname}/requirement/#{fn}", s)
+      end
+    end
+  end
+end

data/lib/metanorma/input/asciidoc.rb

@@ -56,13 +56,19 @@ module Metanorma
         attr&.sub(/^#{name}:\s*$/, "#{name}: true")&.sub(/^#{name}:\s+/, "")
       end
 
-      ADOC_OPTIONS = %w(htmlstylesheet htmlcoverpage htmlintropage scripts
-                        scripts-override scripts-pdf wordstylesheet i18nyaml
-                        standardstylesheet header wordcoverpage wordintropage
-                        ulstyle olstyle htmlstylesheet-override bare
-                        htmltoclevels doctoclevels sectionsplit base-asset-path
-                        body-font header-font monospace-font title-font
-                        align-cross-elements wordstylesheet-override).freeze
+      ADOC_OPTIONS =
+        %w(htmlstylesheet htmlcoverpage htmlintropage scripts
+           scripts-override scripts-pdf wordstylesheet i18nyaml
+           standardstylesheet header wordcoverpage wordintropage
+           ulstyle olstyle htmlstylesheet-override bare
+           htmltoclevels doctoclevels sectionsplit base-asset-path
+           body-font header-font monospace-font title-font
+           align-cross-elements wordstylesheet-override
+           pdf-encrypt pdf-encryption-length pdf-user-password
+           pdf-owner-password pdf-allow-copy-content pdf-allow-edit-content
+           pdf-allow-assemble-document pdf-allow-edit-annotations
+           pdf-allow-print pdf-allow-print-hq pdf-allow-fill-in-forms
+           pdf-allow-access-content pdf-encrypt-metadata).freeze
 
       def extract_options(file)
         header = file.sub(/\n\n.*$/m, "\n")

data/lib/metanorma/registry.rb

@@ -1,7 +1,7 @@
 # Registry of all Metanorma types and entry points
 #
 
-require 'singleton'
+require "singleton"
 
 class Error < StandardError
 end
@@ -14,15 +14,16 @@ module Metanorma
 
     def initialize
       @processors = {}
-      @aliases = {csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa}
+      @aliases = { csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa }
     end
 
     def alias(x)
       @aliases[x]
     end
 
-    def register processor
+    def register(processor)
       raise Error unless processor < ::Metanorma::Processor
+
       p = processor.new
       # p.short[-1] is the canonical name
       short = Array(p.short)
@@ -31,7 +32,8 @@ module Metanorma
         @aliases[s] = short[-1]
       end
       Array(p.short)
-      Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered", :info)
+      Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered",
+               :info)
     end
 
     def find_processor(short)
@@ -42,21 +44,17 @@ module Metanorma
       @processors.keys
     end
 
-    def processors
-      @processors
-    end
-
     def output_formats
-      @processors.inject({}) do |acc, (k,v)|
+      @processors.inject({}) do |acc, (k, v)|
         acc[k] = v.output_formats
         acc
       end
     end
 
     def root_tags
-      @processors.inject({}) do |acc, (k,v)|
+      @processors.inject({}) do |acc, (k, v)|
         if v.asciidoctor_backend
-          x = Asciidoctor.load nil, {backend: v.asciidoctor_backend}
+          x = Asciidoctor.load nil, { backend: v.asciidoctor_backend }
           acc[k] = x.converter.xml_root_tag
         end
         acc

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.4.0".freeze
+  VERSION = "1.4.4".freeze
 end

data/lib/metanorma/worker_pool.rb

@@ -0,0 +1,29 @@
+module Metanorma
+  class WorkersPool
+    def initialize(workers)
+      @workers = workers
+      @queue = SizedQueue.new(@workers)
+      @threads = Array.new(@workers) do
+        Thread.new do
+          catch(:exit) do
+            loop do
+              job, args = @queue.pop
+              job.call *args
+            end
+          end
+        end
+      end
+    end
+
+    def schedule(*args, &block)
+      @queue << [block, args]
+    end
+
+    def shutdown
+      @workers.times do
+        schedule { throw :exit }
+      end
+      @threads.map(&:join)
+    end
+  end
+end

data/metanorma.gemspec

@@ -34,13 +34,14 @@ Gem::Specification.new do |spec|
   # spec.add_dependency "relaton-cli"
   # spec.add_dependency "metanorma-standoc"
 
-  spec.add_development_dependency "byebug", "~> 10.0"
+  spec.add_development_dependency "debug"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "metanorma-iso", "~> 1.9.0"
+  spec.add_development_dependency "metanorma-iso", "~> 1.10"
+  spec.add_development_dependency "mnconvert"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-command", "~> 1.0"
   spec.add_development_dependency "rubocop", "~> 1.5.2"
   spec.add_development_dependency "sassc", "~> 2.4.0"
-  spec.add_development_dependency "mnconvert"
+  spec.add_development_dependency "reline", "~> 0.2.8.pre.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-25 00:00:00.000000000 Z
+date: 2021-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -109,19 +109,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: mnconvert
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -221,19 +235,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 2.4.0
 - !ruby/object:Gem::Dependency
-  name: mnconvert
+  name: reline
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.8.pre.11
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.8.pre.11
 description: Library to process any Metanorma standard.
 email:
 - open.source@ribose.com
@@ -274,6 +288,7 @@ files:
 - lib/metanorma/compile_validate.rb
 - lib/metanorma/config.rb
 - lib/metanorma/document.rb
+- lib/metanorma/extract.rb
 - lib/metanorma/fontist_utils.rb
 - lib/metanorma/input.rb
 - lib/metanorma/input/asciidoc.rb
@@ -283,6 +298,7 @@ files:
 - lib/metanorma/sectionsplit.rb
 - lib/metanorma/util.rb
 - lib/metanorma/version.rb
+- lib/metanorma/worker_pool.rb
 - metanorma.gemspec
 homepage: https://github.com/metanorma/metanorma
 licenses:
@@ -303,7 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Metanorma is the standard of standards; the metanorma gem allows you to create