metanorma 2.1.4 → 2.2.0

data/lib/metanorma/compile/compile.rb

@@ -1,21 +1,38 @@
+# frozen_string_literal: true
+
 require "fileutils"
 require "nokogiri"
 require "htmlentities"
 require "yaml"
 require "fontist"
 require "fontist/manifest/install"
-require_relative "compile_validate"
+require "metanorma-taste"
+require_relative "writeable"
+require_relative "validator"
 require_relative "compile_options"
 require_relative "../util/fontist_helper"
 require_relative "../util/util"
 require_relative "extract"
 require_relative "../collection/sectionsplit/sectionsplit"
 require_relative "../util/worker_pool"
+require_relative "output_filename"
+require_relative "output_filename_config"
+require_relative "flavor"
+require_relative "relaton_drop"
+require_relative "render"
 
 module Metanorma
   class Compile
+    include Validator
+    include CompileOptions
+    include Flavor
+    include Writeable
+
+    DEFAULT_NUM_WORKERS = 3
+
     # @return [Array<String>]
-    attr_reader :errors, :processor
+    attr_reader :errors
+    attr_reader :processor
 
     def initialize
       @registry = Metanorma::Registry.instance
@@ -25,22 +42,48 @@ module Metanorma
       @log = Metanorma::Utils::Log.new
     end
 
+    # Main compile method that orchestrates the document conversion process
+    # @param filename [String] path to the input file
+    # @param options [Hash] compilation options
     def compile(filename, options = {})
-      options_process(filename, options)
+      process_options!(filename, options)
       @processor = @registry.find_processor(options[:type].to_sym)
-      (file, isodoc = process_input(filename, options)) or return nil
-      extensions = get_extensions(options) or return nil
-      relaton_export(isodoc, options)
-      extract(isodoc, options[:extract], options[:extract_type])
-      process_exts(filename, extensions, file, isodoc, options)
+
+      # Step 1: Generate Semantic XML
+      semantic_result = generate_semantic_xml(filename, options)
+      return nil unless semantic_result
+
+      source_file, semantic_xml = semantic_result
+
+      # Step 2: Prepare output paths
+      xml = Nokogiri::XML(semantic_xml, &:huge)
+      bibdata = extract_relaton_metadata(xml)
+      output_paths = prepare_output_paths(filename, bibdata, options)
+
+      # Step 3: Determine which output formats to generate
+      extensions = get_extensions(options)
+      return nil unless extensions
+
+      # Step 4: Extract information from Semantic XML if requested
+      extract_information(semantic_xml, bibdata, options)
+
+      # Step 5: Generate output formats from Semantic XML
+      generate_outputs(
+        source_file,
+        semantic_xml,
+        bibdata,
+        extensions,
+        output_paths,
+        options,
+      )
     ensure
       clean_exit(options)
     end
 
-    def options_process(filename, options)
+    def process_options!(filename, options)
       require_libraries(options)
-      options = options_extract(filename, options)
-      validate_options(options)
+      options = extract_options(filename, options)
+      validate_options!(options)
       @log.save_to(filename, options[:output_dir])
       options[:log] = @log
     end
@@ -50,7 +93,11 @@ module Metanorma
       @log.write
     end
 
-    def process_input(filename, options)
+    # Step 1: Generate Semantic XML from input file
+    # @param filename [String] input file path
+    # @param options [Hash] compilation options
+    # @return [Array, nil] tuple of [source_file, semantic_xml] or nil on failure
+    def generate_semantic_xml(filename, options)
       case extname = File.extname(filename)
       when ".adoc" then process_input_adoc(filename, options)
       when ".xml" then process_input_xml(filename, options)
@@ -61,184 +108,113 @@ module Metanorma
       end
     end
 
-    def process_input_adoc(filename, options)
-      Util.log("[metanorma] Processing: AsciiDoc input.", :info)
-      file = read_file(filename)
-      options[:asciimath] and
-        file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
-      dir = File.dirname(filename)
-      dir != "." and
-        file = file.gsub(/^include::/, "include::#{dir}/")
-          .gsub(/^embed::/, "embed::#{dir}/")
-      [file, @processor.input_to_isodoc(file, filename, options)]
-    end
-
-    def process_input_xml(filename, _options)
-      Util.log("[metanorma] Processing: Metanorma XML input.", :info)
-      # TODO NN: this is a hack -- we should provide/bridge the
-      # document attributes in Metanorma XML
-      ["", read_file(filename)]
-    end
-
-    def read_file(filename)
-      File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
-    end
-
-    def relaton_export(isodoc, options)
-      return unless options[:relaton]
-
-      xml = Nokogiri::XML(isodoc, &:huge)
-      bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
-      # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
-      # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
-      File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
-    end
-
-    def export_output(fname, content, **options)
-      mode = options[:binary] ? "wb" : "w:UTF-8"
-      File.open(fname, mode) { |f| f.write content }
-    end
-
-    def wrap_html(options, file_extension, outfilename)
-      if options[:wrapper] && /html$/.match(file_extension)
-        outfilename = outfilename.sub(/\.html$/, "")
-        FileUtils.mkdir_p outfilename
-        FileUtils.mv "#{outfilename}.html", outfilename
-        FileUtils.mv "#{outfilename}_images", outfilename, force: true
+    # Step 2: Prepare output paths for generated files
+    # Use default filename template if empty string is provided.
+    #
+    # @param filename [String] input file path
+    # @param bibdata [Nokogiri::XML::Element] the bibliographic data element
+    # @param options [Hash] compilation options
+    # @return [Hash] paths for different output formats
+    def prepare_output_paths(filename, bibdata, options)
+      basename = if options[:filename_template].nil?
+                   filename.sub(/\.[^.]+$/, "")
+                 else
+                   drop = RelatonDrop.new(bibdata)
+                   config = OutputFilenameConfig.new(options[:filename_template])
+                   config.generate_filename(drop)
+                 end
+      @output_filename = OutputFilename.new(
+        basename,
+        options[:output_dir],
+        @processor,
+      )
+      {
+        xml: @output_filename.semantic_xml,
+        orig_filename: filename,
+        presentationxml: @output_filename.presentation_xml,
+      }
+    end
+
+    # Step 4: Extract information from Semantic XML
+    # @param semantic_xml [String] semantic XML content
+    # @param options [Hash] compilation options
+    def extract_information(semantic_xml, bibdata, options)
+      # Extract Relaton bibliographic data
+      export_relaton_from_bibdata(bibdata, options) if options[:relaton]
+
+      # Extract other components (sourcecode, images, requirements)
+      if options[:extract]
+        Extract.extract(
+          semantic_xml,
+          options[:extract],
+          options[:extract_type],
+        )
       end
     end
 
-    # isodoc is Raw Metanorma XML
-    def process_exts(filename, extensions, file, isodoc, options)
-      f = File.expand_path(change_output_dir(options))
-      fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
-                 orig_filename: File.expand_path(filename),
-                 presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
+    # Step 5: Generate output formats from Semantic XML
+    # @param source_file [String] source file content
+    # @param semantic_xml [String] semantic XML content
+    # @param bibdata [Nokogiri::XML::Element] the bibliographic data element
+    # @param extensions [Array<Symbol>] output formats to generate
+    # @param output_paths [Hash] paths for output files
+    # @param options [Hash] compilation options
+    def generate_outputs(
+      source_file, semantic_xml, bibdata, extensions, output_paths, options
+    )
       if extensions == %i(presentation)
-        process_ext(:presentation, file, isodoc, fnames, options)
+        # Just generate presentation XML
+        generate_presentation_xml(
+          source_file, semantic_xml, bibdata, output_paths, options
+        )
       else
-      process_exts_queue(fnames, extensions, file, isodoc, options)
+        # Generate multiple output formats with parallel processing
+        generate_outputs_parallel(
+          source_file, semantic_xml, bibdata, extensions, output_paths, options
+        )
       end
     end
 
-    def process_exts_queue(fnames, extensions, file, isodoc, options)
-      @queue = ::Metanorma::Util::WorkersPool
-        .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
-      gather_and_install_fonts(file, options.dup, extensions)
-      process_exts_run(fnames, file, isodoc, extensions, options)
-      @queue.shutdown
+    def process_input_adoc_hdr(file, options)
+      hdr, rest = Metanorma::Input::Asciidoc.new.header(file)
+      attrs = hdr.split("\n")
+      options[:asciimath] and attrs << ":mn-keep-asciimath:"
+      process_input_adoc_overrides(attrs, options)
+      "#{attrs.join("\n")}\n\n#{rest}"
     end
 
-    def process_exts_run(fnames, file, isodoc, extensions, options)
-      Util.sort_extensions_execution(extensions).each do |ext|
-        process_ext(ext, file, isodoc, fnames, options) or break
-      end
+    def process_input_adoc_overrides(attrs, options)
+      @registry.tastes.available_tastes.include?(options[:supplied_type]) or
+        return
+      c = Metanorma::TasteRegister.get(options[:supplied_type])
+      c.process_input_adoc_overrides(attrs, options)
     end
 
-    def gather_and_install_fonts(file, options, extensions)
-      Util.sort_extensions_execution(extensions).each do |ext|
-        isodoc_options = get_isodoc_options(file, options, ext)
-        font_install(isodoc_options.merge(options))
-      end
-    end
-
-    def process_ext(ext, file, isodoc, fnames, options)
-      fnames[:ext] = @processor.output_formats[ext]
-      fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
-      isodoc_options = get_isodoc_options(file, options, ext)
-      thread = true
-      unless process_ext_simple(ext, isodoc, fnames, options,
-                                isodoc_options)
-        thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
-      end
-      thread
-    end
-
-    def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
-      if ext == :rxl
-        relaton_export(isodoc, options.merge(relaton: fnames[:out]))
-      elsif options[:passthrough_presentation_xml] && ext == :presentation
-        #f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
-        f = File.exist?(fnames[:orig_filename]) ? fnames[:orig_filename] : fnames[:f]
-        FileUtils.cp f, fnames[:presentationxml]
-      elsif ext == :html && options[:sectionsplit]
-        sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
-                             isodoc_options)
-      else return false
-      end
-      true
-    end
-
-    def process_exts1(ext, fnames, isodoc, options, isodoc_options)
-      if @processor.use_presentation_xml(ext)
-        @queue.schedule(ext, fnames.dup, options.dup,
-                        isodoc_options.dup) do |a, b, c, d|
-          process_output_threaded(a, b, c, d)
-        end
-      else
-        process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
-      end
-    end
-
-    def process_output_threaded(ext, fnames1, options1, isodoc_options1)
-      @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
-                        isodoc_options1)
-      wrap_html(options1, fnames1[:ext], fnames1[:out])
-    rescue StandardError => e
-      strict = ext == :presentation || isodoc_options1[:strict] == true
-      isodoc_error_process(e, strict, false)
-    end
-
-    def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
-      @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
-                        isodoc_options)
-      true # return as Thread
-    rescue StandardError => e
-      strict = ext == :presentation || isodoc_options[:strict] == "true"
-      isodoc_error_process(e, strict, true)
-      ext != :presentation
+    def process_input_adoc_includes(file, filename)
+      dir = File.dirname(filename)
+      dir != "." and
+        file = file.gsub(/^include::/, "include::#{dir}/")
+          .gsub(/^embed::/, "embed::#{dir}/")
+      file
     end
 
-    # assume we pass in Presentation XML, but we want to recover Semantic XML
-    def sectionsplit_convert(input_filename, file, output_filename = nil,
-                             opts = {})
-      @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
-      input_filename += ".xml" unless input_filename.match?(/\.xml$/)
-      File.exist?(input_filename) or
-        File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
-      presxml = File.read(input_filename, encoding: "utf-8")
-      _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
-
-      ::Metanorma::Collection::Sectionsplit.new(
-        input: input_filename,
-        isodoc: @isodoc,
-        xml: presxml,
-        base: File.basename(output_filename || filename),
-        output: output_filename || filename,
-        dir: dir,
-        compile_opts: opts,
-      ).build_collection
+    def process_input_adoc(filename, options)
+      Util.log("[metanorma] Processing: AsciiDoc input.", :info)
+      file = read_file(filename)
+      file = process_input_adoc_hdr(file, options)
+      file = process_input_adoc_includes(file, filename)
+      [file, @processor.input_to_isodoc(file, filename, options)]
     end
 
-    private
-
-    def isodoc_error_process(err, strict, must_abort)
-      if strict || err.message.include?("Fatal:")
-        @errors << err.message
-      else
-        puts err.message
-      end
-      puts err.backtrace.join("\n")
-      must_abort and 1
+    def process_input_xml(filename, _options)
+      Util.log("[metanorma] Processing: Metanorma XML input.", :info)
+      # TODO NN: this is a hack -- we should provide/bridge the
+      # document attributes in Metanorma XML
+      ["", read_file(filename)]
     end
 
-    # @param options [Hash]
-    # @return [String]
-    def change_output_dir(options)
-      if options[:output_dir]
-        File.join options[:output_dir], File.basename(options[:filename])
-      else options[:filename]
-      end
+    def read_file(filename)
+      File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
     end
   end
 end

data/lib/metanorma/compile/compile_options.rb

@@ -2,104 +2,130 @@ require "csv"
 
 module Metanorma
   class Compile
-    def require_libraries(options)
-      options&.dig(:require)&.each { |r| require r }
-    end
+    module CompileOptions
+      def require_libraries(options)
+        options&.dig(:require)&.each { |r| require r }
+      end
 
-    def xml_options_extract(file)
-      xml = Nokogiri::XML(file, &:huge)
-      if xml.root
-        @registry.root_tags.each do |k, v|
-          return { type: k } if v == xml.root.name
+      def extract_xml_options(file)
+        xml = Nokogiri::XML(file, &:huge)
+        if xml.root
+          @registry.root_tags.each do |k, v|
+            return { type: k } if v == xml.root.name
+          end
         end
+        {}
       end
-      {}
-    end
 
-    def options_extract(filename, options)
-      content = read_file(filename)
-      o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
-        .merge(xml_options_extract(content))
-      options[:type] ||= o[:type]&.to_sym
-      t = @registry.alias(options[:type]) and options[:type] = t
-      dir = filename.sub(%r(/[^/]+$), "/")
-      options[:relaton] ||= File.join(dir, o[:relaton]) if o[:relaton]
-      options[:sourcecode] ||= File.join(dir, o[:sourcecode]) if o[:sourcecode]
-      options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
-      options[:extension_keys] = nil if options[:extension_keys] == [:all]
-      options[:format] ||= :asciidoc
-      options[:filename] = filename
-      options[:fontlicenseagreement] ||= "no-install-fonts"
-      options[:novalid] = o[:novalid] if o[:novalid]
-      options
-    end
+      def extract_options(filename, options)
+        o = options_in_file(filename)
+        extract_flavor_options(options, o)
+        extract_dir_options(options, o, filename)
+        options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
+        options[:extension_keys] = nil if options[:extension_keys] == [:all]
+        options[:format] ||= :asciidoc
+        options[:filename] = filename
+        options[:fontlicenseagreement] ||= "no-install-fonts"
+        options[:novalid] = o[:novalid] if o[:novalid]
+        options
+      end
 
-    def get_extensions(options)
-      ext = extract_extensions(options)
-      !ext.include?(:presentation) && ext.any? do |e|
-        @processor.use_presentation_xml(e)
-      end and ext << :presentation
-      !ext.include?(:rxl) && options[:site_generate] and
-        ext << :rxl
-      ext
-    end
+      def options_in_file(filename)
+        content = read_file(filename)
+        Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
+          .merge(extract_xml_options(content))
+      end
+
+      def extract_flavor_options(options, options_in_file)
+        options[:type] ||= options_in_file[:type]
+        options[:type] = options[:type]&.to_sym
+        options[:supplied_type] = options[:type]
+        t = @registry.alias(options[:type]) and options[:type] = t
+      end
+
+      def extract_dir_options(options, options_in_file, filename)
+        dir = filename.sub(%r(/[^/]+$), "/")
+        options_in_file[:relaton] and
+          options[:relaton] ||= File.join(dir,
+                                          options_in_file[:relaton])
+        options_in_file[:sourcecode] and
+          options[:sourcecode] ||= File.join(dir,
+                                             options_in_file[:sourcecode])
+      end
+
+      def get_extensions(options)
+        ext = extract_extensions(options)
+        !ext.include?(:presentation) && ext.any? do |e|
+          @processor.use_presentation_xml(e)
+        end and ext << :presentation
+        !ext.include?(:rxl) && options[:site_generate] and
+          ext << :rxl
+        ext
+      end
 
-    def extract_extensions(options)
-      options[:extension_keys] ||=
-        @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
-      options[:extension_keys].reduce([]) do |memo, e|
-        if @processor.output_formats[e] then memo << e
-        else
-          unsupported_format_error(e)
-          memo
+      def extract_extensions(options)
+        options[:extension_keys] ||=
+          @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
+        options[:extension_keys].reduce([]) do |memo, e|
+          if @processor.output_formats[e] then memo << e
+          else
+            unsupported_format_error(e)
+            memo
+          end
         end
       end
-    end
 
-    def font_install(opt)
-      @fontist_installed or
-        Util::FontistHelper.install_fonts(@processor, opt)
-      @fontist_installed = true
-    end
+      def font_install(opt)
+        @fontist_installed or
+          Util::FontistHelper.install_fonts(@processor, opt)
+        @fontist_installed = true
+      end
 
-    private
+      private
 
-    def unsupported_format_error(ext)
-      message = "[metanorma] Error: #{ext} format is not supported " \
-                "for this standard."
-      @errors << message
-      Util.log(message, :error)
-    end
+      def unsupported_format_error(ext)
+        message = "[metanorma] Error: #{ext} format is not supported " \
+          "for this standard."
+        @errors << message
+        Util.log(message, :error)
+      end
+
+      def get_isodoc_options(file, options, ext)
+        ret = @processor.extract_options(file)
+        get_isodoc_i18nyaml(options, ret)
+        copy_isodoc_options_attrs(options, ret)
+        font_manifest_mn2pdf(options, ret, ext)
+        ret[:output_formats]&.select! do |k, _|
+          options[:extension_keys].include?(k)
+        end
+        ret[:log] = @log
+        ret
+      end
 
-    def get_isodoc_options(file, options, ext)
-      ret = @processor.extract_options(file)
-      dir = options[:filename].sub(%r(/[^/]+$), "/")
-      ret[:i18nyaml] &&= File.join(dir, ret[:i18nyaml])
-      copy_isodoc_options_attrs(options, ret)
-      font_manifest_mn2pdf(options, ret, ext)
-      ret[:output_formats]&.select! do |k, _|
-        options[:extension_keys].include?(k)
+      def get_isodoc_i18nyaml(options, ret)
+        dir = File.dirname(options[:filename])
+        ret[:i18nyaml] or return
+        (Pathname.new ret[:i18nyaml]).absolute? or
+          ret[:i18nyaml] = File.join(dir, ret[:i18nyaml])
       end
-      ret[:log] = @log
-      ret
-    end
 
-    def copy_isodoc_options_attrs(options, ret)
-      ret[:datauriimage] = true if options[:datauriimage]
-      ret[:sourcefilename] = options[:filename]
-      %i(bare sectionsplit install_fonts baseassetpath aligncrosselements
-         tocfigures toctables tocrecommendations strict)
-        .each { |x| ret[x] ||= options[x] }
-    end
+      def copy_isodoc_options_attrs(options, ret)
+        ret[:datauriimage] = true if options[:datauriimage]
+        ret[:sourcefilename] = options[:filename]
+        %i(bare sectionsplit install_fonts baseassetpath aligncrosselements
+           tocfigures toctables tocrecommendations strict)
+          .each { |x| ret[x] ||= options[x] }
+      end
 
-    def font_manifest_mn2pdf(options, ret, ext)
-      custom_fonts = Util::FontistHelper
-        .has_custom_fonts?(@processor, options, ret)
+      def font_manifest_mn2pdf(options, ret, ext)
+        custom_fonts = Util::FontistHelper
+          .has_custom_fonts?(@processor, options, ret)
 
-      ext == :pdf && custom_fonts and
-        ret[:mn2pdf] = {
-          font_manifest: Util::FontistHelper.location_manifest(@processor, ret),
-        }
+        ext == :pdf && custom_fonts and
+          ret[:mn2pdf] = {
+            font_manifest: Util::FontistHelper.location_manifest(@processor, ret),
+          }
+      end
     end
   end
 end