metanorma 2.1.4 → 2.1.6

data/lib/metanorma/compile/compile.rb

@@ -1,21 +1,36 @@
+# frozen_string_literal: true
+
 require "fileutils"
 require "nokogiri"
 require "htmlentities"
 require "yaml"
 require "fontist"
 require "fontist/manifest/install"
-require_relative "compile_validate"
+require_relative "writeable"
+require_relative "validator"
 require_relative "compile_options"
 require_relative "../util/fontist_helper"
 require_relative "../util/util"
 require_relative "extract"
 require_relative "../collection/sectionsplit/sectionsplit"
 require_relative "../util/worker_pool"
+require_relative "output_filename"
+require_relative "output_filename_config"
+require_relative "flavor"
+require_relative "relaton_drop"
 
 module Metanorma
   class Compile
+    include Validator
+    include CompileOptions
+    include Flavor
+    include Writeable
+
+    DEFAULT_NUM_WORKERS = 3
+
     # @return [Array<String>]
-    attr_reader :errors, :processor
+    attr_reader :errors
+    attr_reader :processor
 
     def initialize
       @registry = Metanorma::Registry.instance
@@ -25,22 +40,48 @@ module Metanorma
       @log = Metanorma::Utils::Log.new
     end
 
+    # Main compile method that orchestrates the document conversion process
+    # @param filename [String] path to the input file
+    # @param options [Hash] compilation options
     def compile(filename, options = {})
-      options_process(filename, options)
+      process_options!(filename, options)
       @processor = @registry.find_processor(options[:type].to_sym)
-      (file, isodoc = process_input(filename, options)) or return nil
-      extensions = get_extensions(options) or return nil
-      relaton_export(isodoc, options)
-      extract(isodoc, options[:extract], options[:extract_type])
-      process_exts(filename, extensions, file, isodoc, options)
+
+      # Step 1: Generate Semantic XML
+      semantic_result = generate_semantic_xml(filename, options)
+      return nil unless semantic_result
+
+      source_file, semantic_xml = semantic_result
+
+      # Step 2: Prepare output paths
+      xml = Nokogiri::XML(semantic_xml, &:huge)
+      bibdata = extract_relaton_metadata(xml)
+      output_paths = prepare_output_paths(filename, bibdata, options)
+
+      # Step 3: Determine which output formats to generate
+      extensions = get_extensions(options)
+      return nil unless extensions
+
+      # Step 4: Extract information from Semantic XML if requested
+      extract_information(semantic_xml, bibdata, options)
+
+      # Step 5: Generate output formats from Semantic XML
+      generate_outputs(
+        source_file,
+        semantic_xml,
+        bibdata,
+        extensions,
+        output_paths,
+        options,
+      )
     ensure
       clean_exit(options)
     end
 
-    def options_process(filename, options)
+    def process_options!(filename, options)
       require_libraries(options)
-      options = options_extract(filename, options)
-      validate_options(options)
+      options = extract_options(filename, options)
+      validate_options!(options)
       @log.save_to(filename, options[:output_dir])
       options[:log] = @log
     end
@@ -50,7 +91,11 @@ module Metanorma
       @log.write
     end
 
-    def process_input(filename, options)
+    # Step 1: Generate Semantic XML from input file
+    # @param filename [String] input file path
+    # @param options [Hash] compilation options
+    # @return [Array, nil] tuple of [source_file, semantic_xml] or nil on failure
+    def generate_semantic_xml(filename, options)
       case extname = File.extname(filename)
       when ".adoc" then process_input_adoc(filename, options)
       when ".xml" then process_input_xml(filename, options)
@@ -61,6 +106,113 @@ module Metanorma
       end
     end
 
+    # Step 2: Prepare output paths for generated files
+    # Use default filename template if empty string is provided.
+    #
+    # @param filename [String] input file path
+    # @param bibdata [Nokogiri::XML::Element] the bibliographic data element
+    # @param options [Hash] compilation options
+    # @return [Hash] paths for different output formats
+    def prepare_output_paths(filename, bibdata, options)
+      basename = if !options[:filename_template].nil?
+                   drop = RelatonDrop.new(bibdata)
+                   config = OutputFilenameConfig.new(options[:filename_template])
+                   config.generate_filename(drop)
+                 else
+                   filename.sub(/\.[^.]+$/, "")
+                 end
+
+      @output_filename = OutputFilename.new(
+        basename,
+        options[:output_dir],
+        @processor,
+      )
+
+      {
+        xml: @output_filename.semantic_xml,
+        orig_filename: filename,
+        presentationxml: @output_filename.presentation_xml,
+      }
+    end
+
+    # Step 4: Extract information from Semantic XML
+    # @param semantic_xml [String] semantic XML content
+    # @param options [Hash] compilation options
+    def extract_information(semantic_xml, bibdata, options)
+      # Extract Relaton bibliographic data
+      export_relaton_from_bibdata(bibdata, options) if options[:relaton]
+
+      # Extract other components (sourcecode, images, requirements)
+      if options[:extract]
+        Extract.extract(
+          semantic_xml,
+          options[:extract],
+          options[:extract_type],
+        )
+      end
+    end
+
+    # Step 5: Generate output formats from Semantic XML
+    # @param source_file [String] source file content
+    # @param semantic_xml [String] semantic XML content
+    # @param bibdata [Nokogiri::XML::Element] the bibliographic data element
+    # @param extensions [Array<Symbol>] output formats to generate
+    # @param output_paths [Hash] paths for output files
+    # @param options [Hash] compilation options
+    def generate_outputs(
+      source_file, semantic_xml, bibdata, extensions, output_paths, options
+    )
+      if extensions == %i(presentation)
+        # Just generate presentation XML
+        generate_presentation_xml(
+          source_file, semantic_xml, bibdata, output_paths, options
+        )
+      else
+        # Generate multiple output formats with parallel processing
+        generate_outputs_parallel(
+          source_file, semantic_xml, bibdata, extensions, output_paths, options
+        )
+      end
+    end
+
+    # Generate presentation XML from semantic XML
+    def generate_presentation_xml(
+      source_file, semantic_xml, bibdata, output_paths, options
+    )
+      process_ext(
+        :presentation, source_file, semantic_xml, bibdata, output_paths, options
+      )
+    end
+
+    # Generate multiple output formats with parallel processing
+    def generate_outputs_parallel(
+      source_file, semantic_xml, bibdata, extensions, output_paths, options
+    )
+      @queue = ::Metanorma::Util::WorkersPool.new(
+        ENV["METANORMA_PARALLEL"]&.to_i || DEFAULT_NUM_WORKERS,
+      )
+
+      # Install required fonts for all extensions
+      gather_and_install_fonts(source_file, options.dup, extensions)
+
+      # Process each extension in order
+      process_extensions_in_order(
+        source_file, semantic_xml, bibdata, extensions, output_paths, options
+      )
+
+      @queue.shutdown
+    end
+
+    def process_extensions_in_order(
+      source_file, semantic_xml, bibdata, extensions, output_paths, options
+    )
+      Util.sort_extensions_execution(extensions).each do |ext|
+        process_ext(
+          ext, source_file, semantic_xml, bibdata, output_paths, options
+        ) or break
+      end
+    end
+
     def process_input_adoc(filename, options)
       Util.log("[metanorma] Processing: AsciiDoc input.", :info)
       file = read_file(filename)
@@ -84,19 +236,21 @@ module Metanorma
       File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
     end
 
-    def relaton_export(isodoc, options)
+    # Export given bibliographic data to Relaton XML on disk
+    # @param bibdata [Nokogiri::XML::Element] the bibliographic data element
+    # @param options [Hash] compilation options
+    def export_relaton_from_bibdata(bibdata, options)
       return unless options[:relaton]
 
-      xml = Nokogiri::XML(isodoc, &:huge)
-      bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
       # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
       # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
-      File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
+      export_output(options[:relaton], bibdata.to_xml)
     end
 
-    def export_output(fname, content, **options)
-      mode = options[:binary] ? "wb" : "w:UTF-8"
-      File.open(fname, mode) { |f| f.write content }
+    # @param xml [Nokogiri::XML::Document] the XML document
+    # @return [Nokogiri::XML::Element] the bibliographic data element
+    def extract_relaton_metadata(xml)
+      xml.at("//bibdata") || xml.at("//xmlns:bibdata")
     end
 
     def wrap_html(options, file_extension, outfilename)
@@ -109,90 +263,101 @@ module Metanorma
     end
 
     # isodoc is Raw Metanorma XML
-    def process_exts(filename, extensions, file, isodoc, options)
-      f = File.expand_path(change_output_dir(options))
-      fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
-                 orig_filename: File.expand_path(filename),
-                 presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
-      if extensions == %i(presentation)
-        process_ext(:presentation, file, isodoc, fnames, options)
-      else
-      process_exts_queue(fnames, extensions, file, isodoc, options)
-      end
-    end
 
-    def process_exts_queue(fnames, extensions, file, isodoc, options)
-      @queue = ::Metanorma::Util::WorkersPool
-        .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
-      gather_and_install_fonts(file, options.dup, extensions)
-      process_exts_run(fnames, file, isodoc, extensions, options)
-      @queue.shutdown
-    end
-
-    def process_exts_run(fnames, file, isodoc, extensions, options)
+    def gather_and_install_fonts(source_file, options, extensions)
       Util.sort_extensions_execution(extensions).each do |ext|
-        process_ext(ext, file, isodoc, fnames, options) or break
-      end
-    end
-
-    def gather_and_install_fonts(file, options, extensions)
-      Util.sort_extensions_execution(extensions).each do |ext|
-        isodoc_options = get_isodoc_options(file, options, ext)
+        isodoc_options = get_isodoc_options(source_file, options, ext)
         font_install(isodoc_options.merge(options))
       end
     end
 
-    def process_ext(ext, file, isodoc, fnames, options)
-      fnames[:ext] = @processor.output_formats[ext]
-      fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
-      isodoc_options = get_isodoc_options(file, options, ext)
-      thread = true
-      unless process_ext_simple(ext, isodoc, fnames, options,
-                                isodoc_options)
-        thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
+    # Process a single extension (output format)
+    def process_ext(ext, source_file, semantic_xml, bibdata, output_paths,
+options)
+      output_paths[:ext] = @processor.output_formats[ext]
+      output_paths[:out] = @output_filename.for_format(ext) ||
+        output_paths[:xml].sub(/\.[^.]+$/, ".#{output_paths[:ext]}")
+      isodoc_options = get_isodoc_options(source_file, options, ext)
+
+      # Handle special cases first
+      return true if process_ext_special(
+        ext, semantic_xml, bibdata, output_paths, options, isodoc_options
+      )
+
+      # Otherwise, determine if it uses presentation XML
+      if @processor.use_presentation_xml(ext)
+        # Format requires presentation XML first, then convert to final format
+        process_via_presentation_xml(ext, output_paths, options, isodoc_options)
+      else
+        # Format can be generated directly from semantic XML
+        process_from_semantic_xml(
+          ext, output_paths, semantic_xml, isodoc_options
+        )
       end
-      thread
     end
 
-    def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
+    # Process special extensions with custom handling
+    def process_ext_special(
+      ext, semantic_xml, bibdata, output_paths, options, isodoc_options
+    )
       if ext == :rxl
-        relaton_export(isodoc, options.merge(relaton: fnames[:out]))
-      elsif options[:passthrough_presentation_xml] && ext == :presentation
-        #f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
-        f = File.exist?(fnames[:orig_filename]) ? fnames[:orig_filename] : fnames[:f]
-        FileUtils.cp f, fnames[:presentationxml]
+
+        # Special case: Relaton export
+        export_relaton_from_bibdata(
+          bibdata,
+          options.merge(relaton: output_paths[:out]),
+        )
+        true
+
+      elsif ext == :presentation && options[:passthrough_presentation_xml]
+
+        # Special case: Pass through presentation XML
+        f = if File.exist?(output_paths[:orig_filename])
+              output_paths[:orig_filename]
+            else
+              output_paths[:xml]
+            end
+
+        FileUtils.cp f, output_paths[:presentationxml]
+        true
+
       elsif ext == :html && options[:sectionsplit]
-        sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
-                             isodoc_options)
-      else return false
+
+        # Special case: Split HTML into sections
+        sectionsplit_convert(
+          output_paths[:xml], semantic_xml, output_paths[:out], isodoc_options
+        )
+        true
+      else
+        false
       end
-      true
     end
 
-    def process_exts1(ext, fnames, isodoc, options, isodoc_options)
-      if @processor.use_presentation_xml(ext)
-        @queue.schedule(ext, fnames.dup, options.dup,
-                        isodoc_options.dup) do |a, b, c, d|
-          process_output_threaded(a, b, c, d)
-        end
-      else
-        process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
+    # Process format that requires presentation XML
+    def process_via_presentation_xml(ext, output_paths, options, isodoc_options)
+      @queue.schedule(ext, output_paths.dup, options.dup,
+                      isodoc_options.dup) do |a, b, c, d|
+        process_output_from_presentation_xml(a, b, c, d)
       end
     end
 
-    def process_output_threaded(ext, fnames1, options1, isodoc_options1)
-      @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
-                        isodoc_options1)
-      wrap_html(options1, fnames1[:ext], fnames1[:out])
+    # Generate output format from presentation XML
+    def process_output_from_presentation_xml(ext, output_paths, options,
+isodoc_options)
+      @processor.output(nil, output_paths[:presentationxml],
+                        output_paths[:out], ext, isodoc_options)
+      wrap_html(options, output_paths[:ext], output_paths[:out])
     rescue StandardError => e
-      strict = ext == :presentation || isodoc_options1[:strict] == true
+      strict = ext == :presentation || isodoc_options[:strict] == true
       isodoc_error_process(e, strict, false)
     end
 
-    def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
-      @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
-                        isodoc_options)
-      true # return as Thread
+    # Process format directly from semantic XML
+    def process_from_semantic_xml(ext, output_paths, semantic_xml,
+isodoc_options)
+      @processor.output(semantic_xml, output_paths[:xml], output_paths[:out],
+                        ext, isodoc_options)
+      true # Return as Thread equivalent
     rescue StandardError => e
       strict = ext == :presentation || isodoc_options[:strict] == "true"
       isodoc_error_process(e, strict, true)
@@ -205,7 +370,7 @@ module Metanorma
       @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
       input_filename += ".xml" unless input_filename.match?(/\.xml$/)
       File.exist?(input_filename) or
-        File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
+        export_output(input_filename, file)
       presxml = File.read(input_filename, encoding: "utf-8")
       _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
 
@@ -231,14 +396,5 @@ module Metanorma
       puts err.backtrace.join("\n")
       must_abort and 1
     end
-
-    # @param options [Hash]
-    # @return [String]
-    def change_output_dir(options)
-      if options[:output_dir]
-        File.join options[:output_dir], File.basename(options[:filename])
-      else options[:filename]
-      end
-    end
   end
 end

data/lib/metanorma/compile/compile_options.rb

@@ -2,104 +2,109 @@ require "csv"
 
 module Metanorma
   class Compile
-    def require_libraries(options)
-      options&.dig(:require)&.each { |r| require r }
-    end
+    module CompileOptions
+      def require_libraries(options)
+        options&.dig(:require)&.each { |r| require r }
+      end
 
-    def xml_options_extract(file)
-      xml = Nokogiri::XML(file, &:huge)
-      if xml.root
-        @registry.root_tags.each do |k, v|
-          return { type: k } if v == xml.root.name
+      def extract_xml_options(file)
+        xml = Nokogiri::XML(file, &:huge)
+        if xml.root
+          @registry.root_tags.each do |k, v|
+            return { type: k } if v == xml.root.name
+          end
         end
+        {}
       end
-      {}
-    end
 
-    def options_extract(filename, options)
-      content = read_file(filename)
-      o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
-        .merge(xml_options_extract(content))
-      options[:type] ||= o[:type]&.to_sym
-      t = @registry.alias(options[:type]) and options[:type] = t
-      dir = filename.sub(%r(/[^/]+$), "/")
-      options[:relaton] ||= File.join(dir, o[:relaton]) if o[:relaton]
-      options[:sourcecode] ||= File.join(dir, o[:sourcecode]) if o[:sourcecode]
-      options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
-      options[:extension_keys] = nil if options[:extension_keys] == [:all]
-      options[:format] ||= :asciidoc
-      options[:filename] = filename
-      options[:fontlicenseagreement] ||= "no-install-fonts"
-      options[:novalid] = o[:novalid] if o[:novalid]
-      options
-    end
+      def extract_options(filename, options)
+        content = read_file(filename)
+        o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
+          .merge(extract_xml_options(content))
+        options[:type] ||= o[:type]&.to_sym
+        t = @registry.alias(options[:type]) and options[:type] = t
+        dir = filename.sub(%r(/[^/]+$), "/")
+        options[:relaton] ||= File.join(dir, o[:relaton]) if o[:relaton]
+        if o[:sourcecode]
+          options[:sourcecode] ||= File.join(dir,
+                                             o[:sourcecode])
+        end
+        options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
+        options[:extension_keys] = nil if options[:extension_keys] == [:all]
+        options[:format] ||= :asciidoc
+        options[:filename] = filename
+        options[:fontlicenseagreement] ||= "no-install-fonts"
+        options[:novalid] = o[:novalid] if o[:novalid]
+        options
+      end
 
-    def get_extensions(options)
-      ext = extract_extensions(options)
-      !ext.include?(:presentation) && ext.any? do |e|
-        @processor.use_presentation_xml(e)
-      end and ext << :presentation
-      !ext.include?(:rxl) && options[:site_generate] and
-        ext << :rxl
-      ext
-    end
+      def get_extensions(options)
+        ext = extract_extensions(options)
+        !ext.include?(:presentation) && ext.any? do |e|
+          @processor.use_presentation_xml(e)
+        end and ext << :presentation
+        !ext.include?(:rxl) && options[:site_generate] and
+          ext << :rxl
+        ext
+      end
 
-    def extract_extensions(options)
-      options[:extension_keys] ||=
-        @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
-      options[:extension_keys].reduce([]) do |memo, e|
-        if @processor.output_formats[e] then memo << e
-        else
-          unsupported_format_error(e)
-          memo
+      def extract_extensions(options)
+        options[:extension_keys] ||=
+          @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
+        options[:extension_keys].reduce([]) do |memo, e|
+          if @processor.output_formats[e] then memo << e
+          else
+            unsupported_format_error(e)
+            memo
+          end
         end
       end
-    end
 
-    def font_install(opt)
-      @fontist_installed or
-        Util::FontistHelper.install_fonts(@processor, opt)
-      @fontist_installed = true
-    end
+      def font_install(opt)
+        @fontist_installed or
+          Util::FontistHelper.install_fonts(@processor, opt)
+        @fontist_installed = true
+      end
 
-    private
+      private
 
-    def unsupported_format_error(ext)
-      message = "[metanorma] Error: #{ext} format is not supported " \
-                "for this standard."
-      @errors << message
-      Util.log(message, :error)
-    end
+      def unsupported_format_error(ext)
+        message = "[metanorma] Error: #{ext} format is not supported " \
+                  "for this standard."
+        @errors << message
+        Util.log(message, :error)
+      end
 
-    def get_isodoc_options(file, options, ext)
-      ret = @processor.extract_options(file)
-      dir = options[:filename].sub(%r(/[^/]+$), "/")
-      ret[:i18nyaml] &&= File.join(dir, ret[:i18nyaml])
-      copy_isodoc_options_attrs(options, ret)
-      font_manifest_mn2pdf(options, ret, ext)
-      ret[:output_formats]&.select! do |k, _|
-        options[:extension_keys].include?(k)
+      def get_isodoc_options(file, options, ext)
+        ret = @processor.extract_options(file)
+        dir = options[:filename].sub(%r(/[^/]+$), "/")
+        ret[:i18nyaml] &&= File.join(dir, ret[:i18nyaml])
+        copy_isodoc_options_attrs(options, ret)
+        font_manifest_mn2pdf(options, ret, ext)
+        ret[:output_formats]&.select! do |k, _|
+          options[:extension_keys].include?(k)
+        end
+        ret[:log] = @log
+        ret
       end
-      ret[:log] = @log
-      ret
-    end
 
-    def copy_isodoc_options_attrs(options, ret)
-      ret[:datauriimage] = true if options[:datauriimage]
-      ret[:sourcefilename] = options[:filename]
-      %i(bare sectionsplit install_fonts baseassetpath aligncrosselements
-         tocfigures toctables tocrecommendations strict)
-        .each { |x| ret[x] ||= options[x] }
-    end
+      def copy_isodoc_options_attrs(options, ret)
+        ret[:datauriimage] = true if options[:datauriimage]
+        ret[:sourcefilename] = options[:filename]
+        %i(bare sectionsplit install_fonts baseassetpath aligncrosselements
+           tocfigures toctables tocrecommendations strict)
+          .each { |x| ret[x] ||= options[x] }
+      end
 
-    def font_manifest_mn2pdf(options, ret, ext)
-      custom_fonts = Util::FontistHelper
-        .has_custom_fonts?(@processor, options, ret)
+      def font_manifest_mn2pdf(options, ret, ext)
+        custom_fonts = Util::FontistHelper
+          .has_custom_fonts?(@processor, options, ret)
 
-      ext == :pdf && custom_fonts and
-        ret[:mn2pdf] = {
-          font_manifest: Util::FontistHelper.location_manifest(@processor, ret),
-        }
+        ext == :pdf && custom_fonts and
+          ret[:mn2pdf] = {
+            font_manifest: Util::FontistHelper.location_manifest(@processor, ret),
+          }
+      end
     end
   end
 end