metanorma-tools 0.1.0

data/lib/metanorma/tools/cli.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module Metanorma
+  module Tools
+    class Cli < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      map %w[--version -v] => :__version
+
+      desc '--version, -v', 'Print the version'
+      def __version
+        puts Metanorma::Tools::VERSION
+      end
+
+      desc 'extract-images INPUT_XML', 'Extract embedded figures from Metanorma presentation XML files'
+      long_desc <<~DESC
+        Extract embedded figures from Metanorma presentation XML files with ISO DRG compliance features.
+
+        This tool automatically extracts document metadata, generates ISO DRG compliant filenames,
+        and supports both SVG and PNG figure formats with optional ZIP packaging.
+
+        For detailed usage examples and options, see: docs/figure-extraction.adoc
+      DESC
+      option :output_dir, type: :string, aliases: '-o', desc: 'Output directory for extracted figures'
+      option :prefix, type: :string, aliases: '-p', desc: 'Prefix for generated figure filenames'
+      option :zip, type: :boolean, default: false, desc: 'Create a ZIP archive of extracted figures'
+      option :verbose, type: :boolean, default: false, aliases: '-v', desc: 'Show detailed progress information'
+      option :retain_original_filenames, type: :boolean, default: false, desc: 'For ISO documents, retain original filenames in generated names'
+
+      def extract_images(input_xml)
+        extractor = FigureExtractor.new(options)
+        extractor.extract(input_xml, options[:output_dir], options[:prefix])
+      end
+
+      # Placeholder for future comment management functionality
+      # Will integrate metanorma/commenter gem functionality
+      desc 'comment SUBCOMMAND', 'Manage ISO comment sheets (planned)'
+      subcommand 'comment', CommentCli if defined?(CommentCli)
+
+      # Placeholder for future ISO document fetching functionality
+      desc 'fetch-iso DOCUMENT_ID', 'Fetch ISO documents from OBP (planned)'
+      def fetch_iso(document_id)
+        puts "ISO document fetching functionality is planned for future release."
+        puts "This will fetch ISO documents from the OBP into Metanorma format."
+        puts "Document ID: #{document_id}"
+      end
+
+      # Help command that shows the expanded purpose
+      desc 'help [COMMAND]', 'Show help for metanorma-tools commands'
+      def help(command = nil)
+        if command.nil?
+          puts <<~HELP
+            Metanorma Tools - Standards Editing Lifecycle Support
+            =====================================================
+
+            Metanorma Tools supports the lifecycle of standards editing for various flavors
+            to facilitate pre and post-compilation of Metanorma documents.
+
+            Available tools:
+            • Figure extraction - Extract embedded figures from Metanorma presentation XML files
+            • Comment management - Manage ISO comment sheets (planned)
+            • ISO document fetching - Fetch documents from OBP (planned)
+
+            For detailed documentation, see the docs/ directory:
+            • docs/figure-extraction.adoc - Figure extraction guide
+            • docs/iso-drg-filename-guidance.adoc - ISO DRG compliance guidance
+
+            Commands:
+          HELP
+        end
+        super
+      end
+    end
+  end
+end

data/lib/metanorma/tools/commands/extract_images.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module Metanorma
+  module Tools
+    module Commands
+      class ExtractImages < Thor
+        desc "extract INPUT_XML", "Extract figures from Metanorma XML"
+        option :output_dir, type: :string, aliases: '-o', desc: 'Output directory for extracted figures'
+        option :prefix, type: :string, aliases: '-p', desc: 'Prefix for generated figure filenames'
+        option :zip, type: :boolean, default: false, desc: 'Create a ZIP archive of extracted figures'
+        option :verbose, type: :boolean, default: false, aliases: '-v', desc: 'Show detailed progress information'
+        option :auto_prefix, type: :boolean, default: true, desc: 'Automatically generate prefix from document metadata'
+
+        def extract(input_xml)
+          extractor = FigureExtractor.new(options)
+          extractor.extract(input_xml, options[:output_dir], options[:prefix])
+        end
+
+        default_task :extract
+      end
+    end
+  end
+end

data/lib/metanorma/tools/commands.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Tools
+    module Commands
+    end
+  end
+end

data/lib/metanorma/tools/document_metadata.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'lutaml/model'
+
+module Metanorma
+  module Tools
+    class DocumentMetadata < Lutaml::Model::Serializable
+      attribute :title, :string
+      attribute :docnumber, :string
+      attribute :stage, :string
+      attribute :substage, :string
+      attribute :docidentifier, :string
+      attribute :standard_number, :string
+      attribute :part_number, :string
+      attribute :edition, :string
+      attribute :stage_code, :string
+      attribute :stage_abbreviation, :string
+      attribute :flavor, :string, default: -> { 'iso' }
+
+      def auto_prefix
+        case flavor
+        when 'iso'
+          # ISO DRG format: {StandardNumber}_{stageCode}_ed{editionNumber}
+          "#{standard_number}_#{stage_abbreviation&.downcase}_ed#{edition}"
+        else
+          # For other flavors, use a generic pattern
+          "#{flavor}_#{standard_number}_#{stage_abbreviation&.downcase}_ed#{edition}"
+        end
+      end
+
+      def to_s
+        if docidentifier
+          "#{docidentifier} - #{title}"
+        else
+          "ISO #{standard_number} Edition #{edition} Stage #{stage_code} (#{stage_abbreviation})"
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/tools/figure.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'fileutils'
+require_relative 'iso_graphic_filename'
+
+module Metanorma
+  module Tools
+    class Figure
+      FORMATS = {
+        datauri_png: { name: 'PNG', ext: 'png', binary: true },
+        datauri_jpeg: { name: 'JPEG', ext: 'jpg', binary: true },
+        datauri_gif: { name: 'GIF', ext: 'gif', binary: true },
+        datauri_svg: { name: 'SVG', ext: 'svg', binary: false },
+        datauri_webp: { name: 'WebP', ext: 'webp', binary: true },
+        svg: { name: 'SVG', ext: 'svg', binary: false }
+      }.freeze
+
+      attr_reader :autonum, :content, :format, :original_filename, :file_size
+
+      def initialize(autonum, content, format, original_filename = nil)
+        @autonum = autonum
+        @content = content
+        @format = format
+        @original_filename = original_filename
+        @file_size = calculate_size
+      end
+
+      def to_file(output_dir, prefix, document_metadata = nil, retain_original_filenames = false)
+        FileUtils.mkdir_p(output_dir)
+
+        filename = generate_filename(prefix, document_metadata, retain_original_filenames)
+        filepath = File.join(output_dir, filename)
+
+        write_content(filepath)
+        puts "  Saved: #{filename}"
+        filepath
+      end
+
+      def format_name
+        FORMATS.dig(@format, :name) || @format.to_s.upcase
+      end
+
+      private
+
+      def calculate_size
+        case @format
+        when *FORMATS.keys.select { |k| k.to_s.start_with?('datauri_') }
+          Base64.decode64(@content).bytesize
+        when :svg
+          @content.bytesize
+        else
+          0
+        end
+      end
+
+      def write_content(filepath)
+        format_info = FORMATS[@format]
+        return unless format_info
+
+        mode = format_info[:binary] ? 'wb' : 'w'
+        encoding = format_info[:binary] ? nil : 'utf-8'
+
+        content_to_write = if @format.to_s.start_with?('datauri_')
+                             Base64.decode64(@content)
+                           else
+                             @content
+                           end
+
+        File.open(filepath, mode, encoding: encoding) do |file|
+          file.write(content_to_write)
+        end
+      end
+
+      def generate_filename(prefix, document_metadata = nil, retain_original_filenames = false)
+        format_info = FORMATS[@format]
+        extension = format_info ? format_info[:ext] : 'unknown'
+
+        # If we have document metadata, use proper ISO DRG filename generation
+        if document_metadata
+          # Parse subfigure from autonum (e.g., "C.2 a" -> figure: "C.2", subfigure: "a")
+          figure_number, subfigure = parse_figure_number(@autonum)
+
+          # Only include original filename if retain_original_filenames is true and we have an original filename
+          original_filename_to_use = (retain_original_filenames && @original_filename && !@original_filename.empty?) ? @original_filename : nil
+
+          iso_filename = IsoGraphicFilename.new(
+            standard_number: document_metadata.standard_number&.to_i,
+            part_number: document_metadata.part_number&.to_i,
+            edition_number: document_metadata.edition&.to_i,
+            stage_code: document_metadata.stage_code,
+            content_type: 'figure',
+            figure_number: figure_number,
+            subfigure: subfigure,
+            file_extension: extension,
+            original_filename: original_filename_to_use
+          )
+
+          return iso_filename.generate_filename
+        end
+
+        # Fallback to simple prefix-based naming
+        sanitized_autonum = @autonum.gsub('.', '')
+        if @original_filename && !@original_filename.empty?
+          basename = File.basename(@original_filename, File.extname(@original_filename))
+          "#{prefix}fig#{sanitized_autonum}_#{basename}.#{extension}"
+        else
+          "#{prefix}fig#{sanitized_autonum}.#{extension}"
+        end
+      end
+
+      def parse_figure_number(autonum)
+        # Handle cases like "C.2 a", "A.1", "3", etc.
+        if autonum.match(/^(.+?)\s+([a-z])$/)
+          # Has subfigure: "C.2 a" -> ["C.2", "a"]
+          [$1, $2]
+        else
+          # No subfigure: "C.2" -> ["C.2", nil]
+          [autonum, nil]
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/tools/figure_extractor.rb

@@ -0,0 +1,384 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'base64'
+require 'fileutils'
+require 'zip'
+require 'tmpdir'
+
+module Metanorma
+  module Tools
+    class FigureExtractor
+      METANORMA_NS = { 'xmlns' => 'https://www.metanorma.org/ns/standoc' }.freeze
+
+      MIMETYPE_FORMATS = {
+        'image/png' => :datauri_png,
+        'image/jpeg' => :datauri_jpeg,
+        'image/jpg' => :datauri_jpeg,
+        'image/gif' => :datauri_gif,
+        'image/svg+xml' => :datauri_svg,
+        'image/webp' => :datauri_webp,
+        '' => :datauri_png # Default for empty mimetype
+      }.freeze
+
+      attr_reader :options
+
+      def initialize(options = {})
+        # Convert string keys to symbols for consistency
+        normalized_options = options.transform_keys(&:to_sym)
+        @options = {
+          zip: false,
+          verbose: false
+        }.merge(normalized_options)
+      end
+
+      def extract(input_xml, output_dir = nil, prefix = nil)
+        validate_input(input_xml)
+
+        doc = parse_xml(input_xml)
+        metadata = extract_document_metadata(doc)
+        prefix = determine_prefix(prefix, metadata)
+        output_dir = determine_output_dir(output_dir, prefix)
+
+        figures = find_figures(doc)
+        return if figures.empty?
+
+        figure_objects, format_counts, total_size = process_figures(figures)
+
+        saved_files = if options[:zip]
+                        extract_to_zip(figure_objects, output_dir, prefix, metadata)
+                      else
+                        extract_to_directory(figure_objects, output_dir, prefix, metadata)
+                      end
+
+        print_summary(metadata, prefix, figure_objects.length, format_counts, total_size, output_dir)
+      end
+
+      private
+
+      def validate_input(input_xml)
+        return if File.exist?(input_xml)
+
+        puts "Error: Input file '#{input_xml}' does not exist."
+        exit 1
+      end
+
+      def parse_xml(input_xml)
+        puts "Reading XML file: #{input_xml}"
+        File.open(input_xml) { |f| Nokogiri::XML(f) }
+      rescue StandardError => e
+        puts "Error processing file: #{e.message}"
+        puts e.backtrace if options[:verbose]
+        exit 1
+      end
+
+      def determine_prefix(prefix, metadata)
+        if prefix.nil? && metadata
+          prefix = metadata.auto_prefix
+          puts "Auto-generated prefix: #{prefix}"
+        end
+
+        if prefix.nil? || prefix.empty?
+          prefix = 'figure'
+          puts "Using default prefix: #{prefix}"
+        end
+
+        prefix
+      end
+
+      def determine_output_dir(output_dir, prefix)
+        if output_dir.nil? || output_dir.empty?
+          if options[:zip]
+            # For ZIP mode, use current directory
+            output_dir = Dir.pwd
+            puts "Using current directory for ZIP output: #{output_dir}"
+          else
+            # For directory mode, use auto-prefix as directory name
+            output_dir = prefix
+            puts "Using auto-generated output directory: #{output_dir}"
+          end
+        end
+        output_dir
+      end
+
+      def extract_to_directory(figure_objects, output_dir, prefix, metadata = nil)
+        # Always extract to temporary directory first, then move to destination
+        Dir.mktmpdir('metanorma_figures_') do |temp_dir|
+          puts "\nExtracting #{figure_objects.length} figures to temporary directory: #{temp_dir}"
+
+          retain_original_filenames = should_retain_original_filenames?(metadata)
+          temp_files = figure_objects.map { |figure_obj| figure_obj.to_file(temp_dir, prefix, metadata, retain_original_filenames) }
+
+          # Ensure output directory exists
+          FileUtils.mkdir_p(output_dir)
+
+          # Move files from temp to final destination
+          puts "Moving files to final destination: #{output_dir}"
+          final_files = []
+          temp_files.each do |temp_file|
+            filename = File.basename(temp_file)
+            final_path = File.join(output_dir, filename)
+            FileUtils.mv(temp_file, final_path)
+            final_files << final_path
+            puts "  Moved: #{filename}"
+          end
+
+          final_files
+        end
+      end
+
+      def extract_to_zip(figure_objects, output_dir, prefix, metadata = nil)
+        # Extract to temporary directory and create ZIP in output directory
+        zip_filename = "#{prefix}.zip"
+        FileUtils.mkdir_p(output_dir) if output_dir
+        zip_path = File.join(output_dir || Dir.pwd, zip_filename)
+
+        Dir.mktmpdir('metanorma_figures_zip_') do |temp_dir|
+          puts "\nExtracting #{figure_objects.length} figures to temporary directory for ZIP: #{temp_dir}"
+
+          retain_original_filenames = should_retain_original_filenames?(metadata)
+          temp_files = figure_objects.map { |figure_obj| figure_obj.to_file(temp_dir, prefix, metadata, retain_original_filenames) }
+
+          puts "Creating ZIP archive: #{zip_filename}"
+          Zip::File.open(zip_path, Zip::File::CREATE) do |zipfile|
+            temp_files.each do |temp_file|
+              filename = File.basename(temp_file)
+              zipfile.add(filename, temp_file)
+              puts "  Added to ZIP: #{filename}"
+            end
+          end
+
+          puts "ZIP archive created: #{zip_path}"
+          [zip_path]
+        end
+      end
+
+      def find_figures(doc)
+        figures = doc.xpath('//xmlns:figure', METANORMA_NS)
+        puts "Found #{figures.length} figures"
+
+        if figures.empty?
+          puts 'No figures found in the document'
+          exit 0
+        end
+
+        figures
+      end
+
+      def process_figures(figures)
+        figure_objects = []
+        format_counts = Hash.new(0)
+        total_size = 0
+
+        figures.each_with_index do |figure_element, index|
+          figure_obj = process_single_figure(figure_element, index)
+          next unless figure_obj
+
+          figure_objects << figure_obj
+          format_counts[figure_obj.format_name] += 1
+          total_size += figure_obj.file_size
+        end
+
+        [figure_objects, format_counts, total_size]
+      end
+
+      def process_single_figure(figure_element, index)
+        autonum = figure_element['autonum']
+
+        unless autonum&.strip&.length&.positive?
+          puts "Warning: Skipping figure #{index + 1} - missing autonum" if options[:verbose]
+          return nil
+        end
+
+        image_element = figure_element.xpath('.//xmlns:image', METANORMA_NS).first
+        unless image_element
+          puts "Warning: Skipping figure #{index + 1} (autonum: #{autonum}) - no image element" if options[:verbose]
+          return nil
+        end
+
+        create_figure_from_image(image_element, autonum)
+      end
+
+      def create_figure_from_image(image_element, autonum)
+        src = image_element['src']
+        filename = image_element['filename']
+        mimetype = image_element['mimetype']
+
+        if src&.start_with?('data:')
+          create_data_uri_figure(src, filename, autonum)
+        elsif mimetype == 'image/svg+xml' || filename&.end_with?('.svg')
+          create_svg_figure(image_element, src, autonum)
+        else
+          log_unsupported_figure(autonum, mimetype, src)
+          nil
+        end
+      end
+
+      def create_data_uri_figure(src, filename, autonum)
+        data_uri_info = parse_data_uri(src)
+        unless data_uri_info
+          puts "Warning: Skipping figure #{autonum} - malformed data URI" if options[:verbose]
+          return nil
+        end
+
+        puts "  Figure #{autonum}: Data URI #{data_uri_info[:format_name]}"
+        Figure.new(autonum, data_uri_info[:content], data_uri_info[:format], filename)
+      end
+
+      def create_svg_figure(image_element, src, autonum)
+        svg_content = image_element.inner_html
+        if svg_content.empty?
+          puts "Warning: Skipping figure #{autonum} - empty SVG content" if options[:verbose]
+          return nil
+        end
+
+        original_filename = src unless src&.start_with?('data:')
+        puts "  Figure #{autonum}: SVG#{original_filename ? " (#{File.basename(original_filename)})" : ''}"
+        Figure.new(autonum, svg_content, :svg, original_filename)
+      end
+
+      def log_unsupported_figure(autonum, mimetype, src)
+        return unless options[:verbose]
+
+        if mimetype && src
+          puts "Warning: Skipping figure #{autonum} - external file not supported: #{File.basename(src)}"
+        else
+          puts "Warning: Skipping figure #{autonum} - no valid source or mimetype found"
+        end
+      end
+
+      def print_summary(metadata, prefix, total_figures, format_counts, total_size, output_dir)
+        puts "\n" + '=' * 60
+        puts 'EXTRACTION SUMMARY'
+        puts '=' * 60
+
+        if metadata
+          puts "Document: #{metadata}"
+          puts "Auto-generated prefix: #{metadata.auto_prefix}" if options[:auto_prefix]
+        end
+
+        puts "File prefix used: #{prefix}"
+        puts "Total figures extracted: #{total_figures}"
+
+        format_counts.each { |format, count| puts "#{format} files: #{count}" }
+
+        puts "Total size: #{format_bytes(total_size)}"
+        puts "Output directory: #{output_dir}"
+        puts "ZIP archive: #{options[:zip] ? 'Created' : 'Not requested'}"
+
+        print_compliance_info(format_counts, metadata)
+        puts '=' * 60
+        puts "\nSuccessfully extracted #{total_figures} figures to #{output_dir}"
+      end
+
+      def print_compliance_info(format_counts, metadata)
+        svg_count = format_counts['SVG'] || 0
+        puts "\nISO DRG COMPLIANCE:"
+        puts "✓ Revisable vector graphics (SVG): #{svg_count > 0 ? 'Yes' : 'No'}"
+        puts '✓ Proper file naming convention: Yes'
+        puts '✓ Language-neutral graphics: Yes (extracted from Metanorma)'
+        puts "✓ Document metadata extraction: #{metadata ? 'Yes' : 'No'}"
+      end
+
+      def extract_document_metadata(doc)
+        # Extract flavor from root metanorma element
+        metanorma_element = doc.xpath('/xmlns:metanorma', METANORMA_NS).first
+        flavor = metanorma_element&.[]('flavor') || 'iso' # Default to 'iso' for compatibility
+
+        bibdata = doc.xpath('//xmlns:bibdata', METANORMA_NS).first
+        return nil unless bibdata
+
+        # Extract basic document information
+        title = xpath_text(bibdata, './/xmlns:title[@type="main"][@language=""]') ||
+                xpath_text(bibdata, './/xmlns:title[@type="main"]') ||
+                xpath_text(bibdata, './/xmlns:title')
+
+        docnumber = xpath_text(bibdata, './/xmlns:docnumber')
+
+        # Extract docidentifier
+        docidentifier = xpath_text(bibdata, './/xmlns:docidentifier[@type="ISO"]') ||
+                       xpath_text(bibdata, './/xmlns:docidentifier')
+
+        # Extract stage information
+        stage_element = bibdata.xpath('.//xmlns:status/xmlns:stage[@language=""]', METANORMA_NS).first
+        stage = stage_element&.text&.strip
+        stage_abbreviation = stage_element&.[]('abbreviation')
+
+        substage = xpath_text(bibdata, './/xmlns:status/xmlns:substage')
+
+        # Extract edition
+        edition = xpath_text(bibdata, './/xmlns:edition[@language=""]') ||
+                 xpath_text(bibdata, './/xmlns:edition')
+
+        # Parse part number from docnumber (e.g., "17301-1" -> part_number: "1")
+        standard_number = docnumber
+        part_number = nil
+        if docnumber&.include?('-')
+          parts = docnumber.split('-', 2)
+          standard_number = parts[0]
+          part_number = parts[1]
+        end
+
+        # Create stage code from stage abbreviation
+        stage_code = stage_abbreviation&.downcase
+
+        DocumentMetadata.new(
+          title: title,
+          docnumber: docnumber,
+          stage: stage,
+          substage: substage,
+          docidentifier: docidentifier,
+          standard_number: standard_number,
+          part_number: part_number,
+          edition: edition,
+          stage_code: stage_code,
+          stage_abbreviation: stage_abbreviation,
+          flavor: flavor
+        )
+      end
+
+      def xpath_text(element, xpath)
+        element.xpath(xpath, METANORMA_NS).first&.text&.strip
+      end
+
+      def parse_data_uri(data_uri)
+        return nil unless data_uri.start_with?('data:')
+
+        uri_content = data_uri[5..-1]
+        parts = uri_content.split(',', 2)
+        return nil if parts.length != 2
+
+        header, data = parts
+        mimetype = header.split(';').first
+        format = MIMETYPE_FORMATS[mimetype.downcase] || :datauri_png
+        format_name = Figure::FORMATS.dig(format, :name) || 'PNG'
+
+        {
+          format: format,
+          format_name: format_name,
+          content: data
+        }
+      rescue StandardError
+        nil
+      end
+
+      def should_retain_original_filenames?(metadata)
+        # Only apply retain_original_filenames option for ISO documents
+        options[:retain_original_filenames] && metadata&.flavor&.downcase == 'iso'
+      end
+
+      def format_bytes(bytes)
+        units = %w[B KB MB GB]
+        size = bytes.to_f
+        unit_index = 0
+
+        while size >= 1024 && unit_index < units.length - 1
+          size /= 1024
+          unit_index += 1
+        end
+
+        "#{size.round(2)} #{units[unit_index]}"
+      end
+    end
+  end
+end