metanorma 2.0.0 → 2.5.5

data/lib/metanorma/compile/render.rb

@@ -0,0 +1,226 @@
+module Metanorma
+  class Compile
+    NO_FONTIST_FORMATS = %i[xml presentation rxl mko].freeze
+
+    # Generate presentation XML from semantic XML
+    def generate_presentation_xml(source_file, xml, bibdata, output_paths, opt)
+      process_ext(:presentation, source_file, xml, bibdata, output_paths, opt)
+    end
+
+    # Generate multiple output formats with parallel processing
+    def generate_outputs_parallel(
+      source_file, semantic_xml, bibdata, extensions, output_paths, options
+    )
+      @queue = ::Metanorma::Util::WorkersPool.new(
+        ENV["METANORMA_PARALLEL"]&.to_i || DEFAULT_NUM_WORKERS,
+      )
+      # Install required fonts for all extensions
+      gather_and_install_fonts(source_file, options.dup, extensions)
+      # Process each extension in order
+      process_extensions_in_order(
+        source_file, semantic_xml, bibdata, extensions, output_paths, options
+      )
+      @queue.shutdown
+    end
+
+    def process_extensions_in_order(
+      source_file, semantic_xml, bibdata, extensions, output_paths, options
+    )
+      Util.sort_extensions_execution(extensions).each do |ext|
+        process_ext(
+          ext, source_file, semantic_xml, bibdata, output_paths, options
+        ) or break
+      end
+    end
+
+    # Export given bibliographic data to Relaton XML on disk
+    # @param bibdata [Nokogiri::XML::Element] the bibliographic data element
+    # @param options [Hash] compilation options
+    def export_relaton_from_bibdata(bibdata, options)
+      options[:relaton] or return
+      export_output(options[:relaton], bibdata.to_xml)
+    end
+
+    # @param xml [Nokogiri::XML::Document] the XML document
+    # @return [Nokogiri::XML::Element] the bibliographic data element
+    def extract_relaton_metadata(xml)
+      xml.at("//bibdata") || xml.at("//xmlns:bibdata")
+    end
+
+    def wrap_html(options, file_extension, outfilename)
+      if options[:wrapper] && /html$/.match(file_extension)
+        outfilename = outfilename.sub(/\.html$/, "")
+        FileUtils.mkdir_p outfilename
+        FileUtils.mv "#{outfilename}.html", outfilename
+        FileUtils.mv "#{outfilename}_images", outfilename, force: true
+      end
+    end
+
+    # isodoc is Raw Metanorma XML
+    def gather_and_install_fonts(source_file, options, extensions)
+      return if (extensions - NO_FONTIST_FORMATS).empty?
+
+      install_options = @processor.extract_options(source_file).merge(options)
+      font_install(install_options)
+    end
+
+    # Process a single extension (output format)
+    def process_ext(ext, source_file, semantic_xml, bibdata, output_paths,
+                    options)
+      output_paths[:ext] = effective_output_formats(options)[ext]
+      output_paths[:out] = @output_filename.for_format(ext) ||
+        output_paths[:xml].sub(/\.[^.]+$/, ".#{output_paths[:ext]}")
+      isodoc_options = get_isodoc_options(source_file, options, ext)
+
+      # Handle special cases first
+      return true if process_ext_special(
+        ext, semantic_xml, bibdata, output_paths, options, isodoc_options
+      )
+
+      # Otherwise, determine if it uses presentation XML
+      if uses_presentation_xml?(ext, options)
+        # Format requires presentation XML first, then convert to final format
+        process_via_presentation_xml(ext, output_paths, options, isodoc_options)
+      else
+        # Format can be generated directly from semantic XML
+        process_from_semantic_xml(
+          ext, output_paths, semantic_xml, isodoc_options
+        )
+      end
+    end
+
+    # Process special extensions with custom handling
+    def process_ext_special(
+      ext, sem_xml, bibdata, output_paths, options, isodoc_options
+    )
+      if ext == :rxl
+        # Special case: Relaton export
+        export_relaton_from_bibdata(
+          bibdata,
+          options.merge(relaton: output_paths[:out]),
+        )
+        true
+      elsif ext == :presentation && options[:passthrough_presentation_xml]
+        # Special case: Pass through presentation XML
+        f = if File.exist?(output_paths[:orig_filename])
+              output_paths[:orig_filename]
+            else
+              output_paths[:xml]
+            end
+        FileUtils.cp f, output_paths[:presentationxml]
+        true
+      elsif ext == :mko
+        # Harness format: export the MKO bundle from the typed model
+        export_mko_bundle(output_paths)
+        true
+      elsif ext == :html && options[:sectionsplit]
+        # Special case: Split HTML into sections
+        sectionsplit_convert(
+          output_paths[:xml], sem_xml, output_paths[:out], isodoc_options
+        )
+        true
+      else
+        false
+      end
+    end
+
+    # Export the MKO machine serialization (metanorma-document). The
+    # semantic and presentation XML are already on disk by the time this
+    # runs; Mko resolves the flavor model via the Core::Flavors registry.
+    def export_mko_bundle(output_paths)
+      require "metanorma/document"
+      require "metanorma/mko"
+      # :xml is the freshly written semantic XML; orig_filename is the
+      # adoc SOURCE and is only right when an output_dir kept the
+      # semantic XML at the input location.
+      semantic_path = [output_paths[:xml], output_paths[:orig_filename]]
+                       .find { |p1| p1 && File.exist?(p1) && File.extname(p1) != ".adoc" }
+      presentation = if File.exist?(output_paths[:presentationxml])
+                       File.read(output_paths[:presentationxml])
+                     end
+      Metanorma::Mko.export(File.read(semantic_path),
+                            to: File.dirname(output_paths[:out]),
+                            presentation_xml: presentation)
+    rescue LoadError => e
+      @log.add_msg("MKO_OUTPUT" => { category: "MKO Output",
+                                     error: "the mko output format requires " \
+                                            "the metanorma-document gem: %s",
+                                     severity: 0 })
+      @log.add("MKO_OUTPUT", nil, params: [e.message])
+      nil
+    end
+
+    # Process format that requires presentation XML
+    def process_via_presentation_xml(ext, output_paths, options, isodoc_options)
+      @queue.schedule(ext, output_paths.dup, options.dup,
+                      isodoc_options.dup) do |a, b, c, d|
+                        process_output_from_presentation_xml(a, b, c, d)
+                      end
+    end
+
+    # Generate output format from presentation XML
+    def process_output_from_presentation_xml(ext, output_paths, options,
+                                             isodoc_options)
+      @processor.output(nil, output_paths[:presentationxml],
+                        output_paths[:out], ext, isodoc_options)
+      wrap_html(options, output_paths[:ext], output_paths[:out])
+    rescue StandardError => e
+      isodoc_error_process(e, strict_error?(ext, isodoc_options), false)
+    end
+
+    # Process format directly from semantic XML
+    def process_from_semantic_xml(ext, output_paths, sem_xml, isodoc_options)
+      @processor.output(sem_xml, output_paths[:xml], output_paths[:out],
+                        ext, isodoc_options)
+      true # Return as Thread equivalent
+    rescue StandardError => e
+      isodoc_error_process(e, strict_error?(ext, isodoc_options), true)
+      ext != :presentation
+    end
+
+    # assume we pass in Presentation XML, but we want to recover Semantic XML
+    def sectionsplit_convert(input_filename, file, output_filename = nil,
+                             opts = {})
+      @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
+      input_filename += ".xml" unless input_filename.match?(/\.xml$/)
+      File.exist?(input_filename) or export_output(input_filename, file)
+      presxml = File.read(input_filename, encoding: "utf-8")
+      _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
+      ::Metanorma::Collection::Sectionsplit.new(
+        input: input_filename, isodoc: @isodoc, xml: presxml,
+        base: File.basename(output_filename || filename),
+        sectionsplit_filename: opts[:sectionsplit_filename],
+        output: output_filename || filename, dir: dir, compile_opts: opts
+      ).build_collection
+    end
+
+    private
+
+    # A presentation failure is always strict: every downstream format
+    # needs the presentation XML. The :strict option arrives as a
+    # boolean from the CLI but as a string from document attributes, so
+    # accept both (the two rescue sites previously each checked only
+    # one form).
+    def strict_error?(ext, isodoc_options)
+      ext == :presentation ||
+        [true, "true"].include?(isodoc_options[:strict])
+    end
+
+    # On the strict path the message is pooled into @errors for the
+    # caller to surface (the standalone CLI reports the pool and
+    # aborts; the collection renderer attributes it to the failed
+    # member, #586). The class+message line is also printed here in all
+    # cases, so it sits adjacent to its backtrace in the build log: a
+    # pooled message surfaces only at end of run, and a bare backtrace
+    # mid-log is undiagnosable. The exception class is retained because
+    # for errors like Encoding::UndefinedConversionError it carries
+    # half the diagnosis.
+    def isodoc_error_process(err, strict, must_abort)
+      msg = "#{err.class}: #{err.message}"
+      strict || err.message.include?("Fatal:") and @errors << msg
+      puts msg
+      puts err.backtrace.join("\n")
+      must_abort and 1
+    end
+  end
+end

data/lib/metanorma/compile/validator.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Metanorma
+  class Compile
+    module Validator
+      def validate_options!(options)
+        validate_type!(options)
+        validate_format!(options)
+      end
+
+      def validate_type!(options)
+        options[:type] or
+          Util.log("[metanorma] Error: Please specify a standard type: "\
+                   "#{@registry.supported_backends}.", :fatal)
+        stdtype = options[:type].to_sym
+        load_flavor(stdtype)
+      end
+
+      def validate_format!(options)
+        options[:format] == :asciidoc or
+          Util.log("[metanorma] Error: Only source file format currently "\
+                   "supported is 'asciidoc'.", :fatal)
+      end
+    end
+  end
+end

data/lib/metanorma/compile/writeable.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Metanorma
+  class Compile
+    module Writeable
+      def export_output(fname, content, **options)
+        mode = options[:binary] ? "wb" : "w:UTF-8"
+        File.open(fname, mode) { |f| f.write content }
+      end
+    end
+  end
+end

data/lib/metanorma/util/fontist_helper.rb

@@ -15,12 +15,12 @@ module Metanorma
 
         def validate_install_fonts(processor, options)
           unless install_fonts?(options)
-            Util.log("[fontist] Skip font installation because" \
-                     " --no-install-fonts argument passed", :debug)
+            Util.log("[fontist] Skip font installation because " \
+                     "--no-install-fonts argument passed", :debug)
             return false
           end
           unless has_custom_fonts?(processor, options, options)
-            Util.log("[fontist] Skip font installation because "\
+            Util.log("[fontist] Skip font installation because " \
                      "fonts_manifest is missing", :debug)
             return false
           end
@@ -34,24 +34,23 @@ module Metanorma
         rescue Fontist::Errors::FontError => e
           log_level = continue ? :warning : :fatal
           Util.log("[fontist] '#{e.font}' font is not supported. " \
-                   "Please report this issue at github.com/metanorma/metanorma" \
-                   "/issues to report this issue.", log_level)
+                   "Please report this issue at " \
+                   "https://github.com/metanorma/metanorma/issues", log_level)
         rescue Fontist::Errors::FormulaIndexNotFoundError
           fontist_update_repo(manifest, agree, continue, no_progress)
         end
 
         def fontist_install(manifest, agree, no_progress)
-          if agree
-            no_license_log
+          if agree then no_license_log
           else
-            Fontist.log_level = :info
+            # Fontist.log_level = :info
+            # Shutting up Fontist 2.0's verbose logging of installation locations
+            Fontist.log_level = :fatal
           end
-
-          Fontist::Manifest::Install.from_hash(
-            manifest,
+          Fontist::Manifest.from_hash(manifest).install(
             confirmation: agree ? "yes" : "no",
             no_progress: no_progress,
-          )
+          ).to_hash
         end
 
         def license_error_log(continue)
@@ -61,8 +60,8 @@ module Metanorma
               :debug,
             )
           else
-            Util.log("[fontist] Aborting without proper fonts installed," \
-                     " make sure that you have set option --agree-to-terms",
+            Util.log("[fontist] Aborting without proper fonts installed, " \
+                     "make sure that you have set option --agree-to-terms",
                      :fatal)
           end
         end
@@ -92,7 +91,12 @@ module Metanorma
         return unless validate_install_fonts(processor, options)
 
         @@updated_formulas_repo = false
-        manifest = processor.fonts_manifest.dup
+        manifest = if processor.respond_to?(:fonts_manifest) &&
+            !processor.fonts_manifest.nil?
+                     processor.fonts_manifest.dup
+                   else
+                     {}
+                   end
         append_source_fonts(manifest, options)
         agree_to_terms, can_without_fonts, no_progress = validate_options(options)
 
@@ -105,16 +109,28 @@ module Metanorma
       end
 
       def self.has_custom_fonts?(processor, options, source_attributes)
-        install_fonts?(options) \
+        (install_fonts?(options) \
           && processor.respond_to?(:fonts_manifest) \
-          && !processor.fonts_manifest.nil? \
+          && !processor.fonts_manifest.nil?) \
           || source_attributes[:fonts]
       end
 
       def self.location_manifest(processor, source_attributes)
-        Fontist::Manifest::Locations.from_hash(
-          append_source_fonts(processor.fonts_manifest.dup, source_attributes),
+        base = if processor.respond_to?(:fonts_manifest) &&
+            !processor.fonts_manifest.nil?
+                 processor.fonts_manifest.dup
+               else
+                 {}
+               end
+
+        return nil if base.empty? && !source_attributes[:fonts]
+
+        instance = Fontist::Manifest.from_hash(
+          append_source_fonts(base, source_attributes),
+          locations: true,
         )
+
+        instance.to_hash unless instance.nil?
       end
 
       def self.append_source_fonts(manifest, source_attributes)

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "2.0.0".freeze
+  VERSION = "2.5.5".freeze
 end

data/lib/metanorma.rb

@@ -5,13 +5,7 @@ class Array
 end
 
 require "metanorma/version"
-require "asciidoctor"
-require "metanorma/util/util"
-require "metanorma/config/config"
-require "metanorma/input"
-require "metanorma/registry/registry"
-require "metanorma/processor/processor"
-require "metanorma/asciidoctor_extensions"
+require "metanorma-core"
 require "metanorma/compile/compile"
 require "metanorma/collection/collection"
 require "metanorma/collection/manifest/manifest"

data/metanorma.gemspec

@@ -15,35 +15,29 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features|bin|.github)/}) \
-    || f.match(%r{Rakefile|bin/rspec})
+    || f.match(%r{Rakefile|bin/rspec}) \
+    || f.match(%r{flake|\.(?:direnv|pryrc|irbrc|nix)})
   end
   spec.extra_rdoc_files = %w[README.adoc CHANGELOG.adoc LICENSE.txt]
   spec.bindir        = "bin"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.7.0"
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0")
 
   spec.add_runtime_dependency "asciidoctor"
-  spec.add_runtime_dependency "fontist", ">= 1.14.3"
+  spec.add_runtime_dependency "concurrent-ruby"
+  spec.add_runtime_dependency "fontist", ">= 2.1.2"
   spec.add_runtime_dependency "htmlentities"
-  spec.add_runtime_dependency "isodoc", ">= 2.6.3"
-  spec.add_runtime_dependency "mn2pdf", "~> 1"
+  spec.add_runtime_dependency "isodoc", ">= 3.5.0"
+  spec.add_runtime_dependency "marcel"
+  spec.add_runtime_dependency "metanorma-core"
+  spec.add_runtime_dependency "metanorma-standoc"
+  spec.add_runtime_dependency "mn2pdf", "~> 2"
   spec.add_runtime_dependency "nokogiri"
-  spec.add_runtime_dependency "shale"
-
-  # get relaton-cli to avoid circular reference with metanorma-standoc
-  # spec.add_dependency "relaton-cli"
-  # spec.add_dependency "metanorma-standoc"
-
-  spec.add_development_dependency "debug"
-  spec.add_development_dependency "equivalent-xml", "~> 0.6"
+  spec.add_development_dependency "canon" #, "= 0.2.3"
+  spec.add_development_dependency "metanorma-iho"
   spec.add_development_dependency "metanorma-iso"
-  spec.add_development_dependency "mnconvert"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "rspec-command", "~> 1.0"
-  spec.add_development_dependency "rubocop", "~> 1.5.2"
-  spec.add_development_dependency "sassc", "~> 2.4.0"
-  spec.add_development_dependency "simplecov", "~> 0.15"
+
+  # relaton-cli is required by Metanorma::Collection
+  spec.add_dependency "relaton-cli"
 end