suma 0.2.5 → 0.2.6

data/lib/suma/cli/generate_schemas.rb

@@ -1,12 +1,11 @@
+# frozen_string_literal: true
+
 require "thor"
 require_relative "../thor_ext"
-require "fileutils"
-require "yaml"
-require "pathname"
+require_relative "../schema_manifest_generator"
 
 module Suma
   module Cli
-    # GenerateSchemas command to generate Schemas YAML by Metanorma YAML
     class GenerateSchemas < Thor
       desc "generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE",
            "Generate EXPRESS schema manifest file from Metanorma site manifest"
@@ -14,125 +13,12 @@ module Suma
                              desc: "Exclude schemas paths by pattern " \
                                    "(e.g. `*_lf.exp`)"
 
-      YAML_FILE_EXTENSIONS = [".yaml", ".yml"].freeze
-
-      def generate_schemas(metanorma_manifest_file, schema_manifest_file) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-        metanorma_manifest_file = File.expand_path(metanorma_manifest_file)
-
-        unless File.exist?(metanorma_manifest_file)
-          raise Errno::ENOENT, "Specified file `#{metanorma_manifest_file}` " \
-                               "not found."
-        end
-
-        unless File.file?(metanorma_manifest_file)
-          raise ArgumentError, "Specified path `#{metanorma_manifest_file}` " \
-                               "is not a file."
-        end
-
-        [metanorma_manifest_file, schema_manifest_file].each do |file|
-          if !YAML_FILE_EXTENSIONS.include?(File.extname(file))
-            raise ArgumentError, "Specified file `#{file}` is not a YAML file."
-          end
-        end
-
-        run(
-          metanorma_manifest_file, schema_manifest_file,
-          exclude_paths: options[:exclude_paths]
-        )
-      end
-
-      private
-
-      def run(metanorma_manifest_file, schema_manifest_file, exclude_paths: nil)
-        metanorma_data = load_yaml(metanorma_manifest_file)
-        collection_files = metanorma_data["metanorma"]["source"]["files"]
-        manifest_files = load_manifest_files(collection_files)
-        all_schemas = load_project_schemas(manifest_files, exclude_paths,
-                                           schema_manifest_file)
-        all_schemas["schemas"] = all_schemas["schemas"].sort.to_h
-        output_data(all_schemas, schema_manifest_file)
-      end
-
-      def output_data(all_schemas, path)
-        puts "Writing the Schemas YAML file to #{File.expand_path(path)}..."
-        # debug use only
-        # puts all_schemas.to_yaml
-        File.write(File.expand_path(path), all_schemas.to_yaml)
-        puts "Writing the Schemas YAML file to #{File.expand_path(path)}...Done"
-      end
-
-      def load_yaml(file_path)
-        YAML.safe_load(
-          File.read(file_path, encoding: "UTF-8"),
-          aliases: true,
-        )
-      end
-
-      def load_manifest_files(collection_files)
-        manifest_files = collection_files.map do |c|
-          collection_data = load_yaml(c)
-          collection_data["manifest"]["docref"].map { |docref| docref["file"] }
-        end
-        manifest_files.flatten
-      end
-
-      def load_project_schemas( # rubocop:disable Metrics/AbcSize
-        manifest_files, exclude_paths, schema_manifest_file
-      )
-        all_schemas = { "schemas" => {} }
-
-        manifest_files.each do |file|
-          # load schemas.yaml from the location of the collection.yml file
-          schemas_file_path = File.expand_path(
-            file.gsub("collection.yml", "schemas.yaml"),
-          )
-          unless File.exist?(schemas_file_path)
-            puts "Schemas file not found: #{schemas_file_path}"
-            next
-          end
-
-          schemas_data = load_yaml(schemas_file_path)
-
-          if schemas_data["schemas"]
-            schemas_data["schemas"] = fix_path(
-              schemas_data,
-              schemas_file_path,
-              schema_manifest_file,
-            )
-            all_schemas["schemas"].merge!(schemas_data["schemas"])
-          end
-
-          if exclude_paths
-            all_schemas["schemas"].delete_if do |_key, value|
-              value["path"].match?(
-                Regexp.new(exclude_paths.gsub("*", "(.*){1,999}")),
-              )
-            end
-          end
-        end
-
-        all_schemas
-      end
-
-      def fix_path(schemas_data, schemas_file_path, schema_manifest_file) # rubocop:disable Metrics/AbcSize
-        schema_manifest_path = File.expand_path(schema_manifest_file, Dir.pwd)
-
-        schemas_data["schemas"].each do |key, value|
-          # resolve the path in the schema file by the path in the schemas.yaml
-          path_in_schema = File.expand_path(
-            value["path"],
-            File.dirname(schemas_file_path),
-          )
-
-          # calculate the relative path from the schema manifest file
-          fixed_path = Pathname.new(path_in_schema).relative_path_from(
-            Pathname.new(File.dirname(schema_manifest_path)),
-          )
-
-          { key => value.merge!("path" => fixed_path.to_s) }
-        end
-
-        schemas_data["schemas"]
+      def generate_schemas(metanorma_manifest_file, schema_manifest_file)
+        SchemaManifestGenerator.new(
+          metanorma_manifest_file,
+          schema_manifest_file,
+          exclude_paths: options[:exclude_paths],
+        ).generate
       end
     end
   end

data/lib/suma/cli/validate_links.rb

@@ -2,11 +2,11 @@
 
 require "thor"
 require_relative "../utils"
+require_relative "../link_validator"
 require "expressir"
 
 module Suma
   module Cli
-    # ValidateLinks command for managing EXPRESS links
     class ValidateLinks < Thor
       desc "extract_and_validate SCHEMAS_FILE DOCUMENTS_PATH [OUTPUT_FILE]",
            "Extract and validate express links without creating intermediate file"
@@ -16,47 +16,38 @@ module Suma
         load_dependencies
         paths = prepare_file_paths(schemas_file, documents_path, output_file)
 
-        # Load schemas and extract links
         schemas_config = load_schemas_config(paths[:schemas_file])
-        exp_files = collect_schema_paths(schemas_config,
-                                         paths[:schemas_file_rel])
+        exp_files = collect_schema_paths(schemas_config, paths[:schemas_file_rel])
         adoc_files = find_adoc_files(paths[:documents_path])
 
         all_files = adoc_files + exp_files
         display_file_counts(adoc_files, exp_files)
 
-        # Extract links from files
         links_by_file = extract_links(all_files)
 
-        # Validate links against schemas
         repo = load_express_schemas(schemas_config)
-        unresolved_links = validate_links(links_by_file, repo)
+        index = SchemaIndex.new(repo)
+        unresolved = LinkValidator.new(index).validate(links_by_file)
 
-        # Generate and output results
         write_validation_results(paths[:output_file], paths[:output_file_rel],
-                                 unresolved_links, links_by_file)
+                                 unresolved, links_by_file)
       end
 
       private
 
-      # Load all required dependencies for link validation
       def load_dependencies
-        # Lazy-load dependencies only when this command is actually used
         require "expressir"
         require "ruby-progressbar"
         require "pathname"
       end
 
-      # Prepare and normalize all file paths needed for the operation
       def prepare_file_paths(schemas_file, documents_path, output_file)
-        # Convert to absolute paths
         schemas_file_path = Pathname.new(schemas_file).expand_path
-        documents_path = Pathname.new(documents_path).expand_path
+        documents_path_exp = Pathname.new(documents_path).expand_path
         output_file_path = Pathname.new(output_file).expand_path
 
-        # Store relative paths for display
         schemas_file_rel = Pathname.new(schemas_file_path).relative_path_from(Pathname.pwd).to_s
-        documents_path_rel = Pathname.new(documents_path).relative_path_from(Pathname.pwd).to_s
+        documents_path_rel = Pathname.new(documents_path_exp).relative_path_from(Pathname.pwd).to_s
         output_file_rel = Pathname.new(output_file_path).relative_path_from(Pathname.pwd).to_s
 
         puts "Extracting and validating express links using schemas from #{schemas_file_rel}..."
@@ -65,46 +56,35 @@ module Suma
         {
           schemas_file: schemas_file_path,
           schemas_file_rel: schemas_file_rel,
-          documents_path: documents_path,
+          documents_path: documents_path_exp,
           documents_path_rel: documents_path_rel,
           output_file: output_file_path,
           output_file_rel: output_file_rel,
         }
       end
 
-      # Load and initialize the schemas configuration
       def load_schemas_config(schemas_file_path)
         schemas_config = Expressir::SchemaManifest.from_yaml(File.read(schemas_file_path))
-        # Ensure the config is initialized with the correct path to resolve relative paths
         schemas_config.set_initial_path(schemas_file_path.to_s)
         schemas_config
       rescue StandardError => e
-        puts "Error loading schemas file: #{e.message}"
-        exit(1)
+        raise Suma::Error, "Error loading schemas file: #{e.message}"
       end
 
-      # Collect paths to all schema files from the config
       def collect_schema_paths(schemas_config, schemas_file_rel)
-        exp_files = []
-        schemas_config.schemas.each do |schema|
-          exp_files << schema.path if schema.path
-        end
-
+        exp_files = schemas_config.schemas.filter_map(&:path)
         puts "Found #{exp_files.size} EXPRESS schema files from #{schemas_file_rel}"
         exp_files
       end
 
-      # Find all AsciiDoc files in the specified path
       def find_adoc_files(documents_path)
         Dir.glob(documents_path.join("**", "*.adoc").to_s)
       end
 
-      # Display counts of discovered files
       def display_file_counts(adoc_files, exp_files)
         puts "Found #{adoc_files.size} AsciiDoc files and #{exp_files.size} EXPRESS files"
       end
 
-      # Create a standardized progress bar
       def create_progress_bar(title, total)
         ProgressBar.create(
           title: title,
@@ -116,7 +96,6 @@ module Suma
         )
       end
 
-      # Extract EXPRESS links from all files
       def extract_links(files)
         links_by_file = {}
         link_count = 0
@@ -127,7 +106,6 @@ module Suma
           progress.increment
           begin
             content = File.read(file)
-            # Extract links while ignoring text after comma
             express_links = content.scan(/<<express:([^,>]+)(?:,[^>]+)?>>/).flatten.uniq
 
             if express_links.any?
@@ -143,282 +121,31 @@ module Suma
         links_by_file
       end
 
-      # Load all EXPRESS schemas for validation
       def load_express_schemas(schemas_config)
-        # Get all schema paths for validation
         schema_paths = {}
-        schemas_config.schemas.each do |schema|
-          schema_paths[schema.id] = schema.path
-        end
+        schemas_config.schemas.each { |s| schema_paths[s.id] = s.path }
 
         puts "Loading #{schema_paths.size} EXPRESS schemas for validation..."
 
-        # Setup progress bar for schema loading
-        loading_progress = create_progress_bar("Loading schemas",
-                                               schema_paths.size)
+        loading_progress = create_progress_bar("Loading schemas", schema_paths.size)
 
-        # Try to load all schemas with progress tracking
         begin
           repo = Expressir::Express::Parser.from_files(schema_paths.values) do |filename, _schemas, error|
             loading_progress.increment
-            if error
-              puts "\nWarning: Error loading schema #{filename}: #{error.message}"
-            end
+            puts "\nWarning: Error loading schema #{filename}: #{error.message}" if error
           end
 
           puts "Successfully loaded #{repo.schemas.size} schemas"
           repo
         rescue StandardError => e
-          puts "Error loading schemas: #{e.message}"
-          exit(1)
-        end
-      end
-
-      # Validate all links against loaded schemas
-      def validate_links(links_by_file, repo)
-        unresolved_links = []
-        total_links = links_by_file.values.sum(&:size)
-
-        progress = create_progress_bar("Validating links", total_links)
-
-        links_by_file.each do |file, links|
-          validate_file_links(file, links, repo, progress, unresolved_links)
-        end
-
-        unresolved_links
-      end
-
-      # Validate links in a specific file
-      def validate_file_links(file, links, repo, progress, unresolved_links)
-        file_content = File.read(file)
-        file_lines = file_content.lines
-
-        links.each do |link|
-          progress.increment
-          line_idx = find_link_line(file_lines, link)
-          next unless line_idx
-
-          # Parse link (schema only, schema.element, or schema.element.path)
-          parts = link.split(".")
-
-          if parts.size == 1
-            validate_schema_only_link(parts[0], repo, file, line_idx, link,
-                                      unresolved_links)
-          else
-            validate_schema_element_link(parts, repo, file, line_idx, link,
-                                         unresolved_links)
-          end
-        end
-      rescue StandardError => e
-        puts "Warning: Error processing file #{file}: #{e.message}"
-      end
-
-      # Find the line where a link appears in the file
-      def find_link_line(file_lines, link)
-        file_lines.each_with_index do |line, idx|
-          # Match both with and without comma text
-          if /<<express:#{Regexp.escape(link)}(?:,[^>]+)?>>/.match?(line)
-            return idx
-          end
-        end
-        nil
-      end
-
-      # Validate a link that only references a schema
-      def validate_schema_only_link(schema_name, repo, file, line_idx, link,
-  unresolved_links)
-        # Check if schema exists
-        schema = repo.schemas.find do |s|
-          s.id.downcase == schema_name.downcase
-        end
-
-        if !schema
-          unresolved_links << {
-            file: file,
-            line: line_idx + 1,
-            link: link,
-            reason: "Schema '#{schema_name}' not found",
-          }
-        end
-      end
-
-      # Validate a link with schema.element or deeper paths
-      def validate_schema_element_link(parts, repo, file, line_idx, link,
-  unresolved_links)
-        schema_name = parts[0]
-        element_name = parts[1]
-
-        # Check if schema exists
-        schema = repo.schemas.find do |s|
-          s.id.downcase == schema_name.downcase
-        end
-
-        if !schema
-          unresolved_links << {
-            file: file,
-            line: line_idx + 1,
-            link: link,
-            reason: "Schema '#{schema_name}' not found",
-          }
-          return
-        end
-
-        # Find the element in the schema
-        element = find_schema_element(schema, element_name)
-
-        if !element
-          unresolved_links << {
-            file: file,
-            line: line_idx + 1,
-            link: link,
-            reason: "Element '#{element_name}' not found in schema '#{schema_name}'",
-          }
-          return
-        end
-
-        # If we have more than 2 parts, validate the deeper path
-        if parts.size > 2
-          validation_error = validate_deep_path(schema, element, parts[2..],
-                                                file, line_idx, link)
-          unresolved_links << validation_error if validation_error
-        end
-      end
-
-      # Find an element in a schema by name
-      def find_schema_element(schema, element_name)
-        # Try to find element in various collections
-        element_collections = [
-          schema.entities,
-          schema.types,
-          schema.constants,
-          schema.functions,
-          schema.rules,
-          schema.procedures,
-          schema.subtype_constraints,
-        ]
-
-        element_collections.each do |collection|
-          element = collection&.find do |e|
-            e.id.downcase == element_name.downcase
-          end
-          return element if element
-        end
-
-        nil
-      end
-
-      # Validate deeper paths in a link (schema.element.path.subpath)
-      def validate_deep_path(schema, element, path_parts, file, line_idx,
-  full_link)
-        current_element = element
-        current_path = "#{schema.id}.#{element.id}"
-
-        # Process each part of the path
-        path_parts.each do |part|
-          # The validation logic depends on the type of current element
-          case current_element
-          when Expressir::Express::Entity
-            # For entities, check attributes
-            attribute = current_element.attributes&.find do |a|
-              a.id.downcase == part.downcase
-            end
-
-            unless attribute
-              return {
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Attribute '#{part}' not found in entity '#{current_path}'",
-              }
-            end
-
-            current_element = attribute
-            current_path += ".#{part}"
-
-          when Expressir::Express::Type
-            # For types, validation depends on type kind
-            if current_element.respond_to?(:base_type) && current_element.base_type
-              # For derived types, find the base type
-              base_type = find_base_type(schema, current_element.base_type)
-
-              unless base_type
-                return {
-                  file: file,
-                  line: line_idx + 1,
-                  link: full_link,
-                  reason: "Base type not found for '#{current_path}'",
-                }
-              end
-
-              # Continue validation using base type
-              current_element = base_type
-
-            elsif current_element.respond_to?(:elements) && current_element.elements
-              # For enum types, check enum values
-              enum_value = current_element.elements.find do |e|
-                e.id.downcase == part.downcase
-              end
-
-              unless enum_value
-                return {
-                  file: file,
-                  line: line_idx + 1,
-                  link: full_link,
-                  reason: "Enumeration value '#{part}' not found in type '#{current_path}'",
-                }
-              end
-
-              current_element = enum_value
-              current_path += ".#{part}"
-
-            else
-              # For other types, we can't navigate deeper
-              return {
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Cannot navigate deeper from type '#{current_path}'",
-              }
-            end
-
-          else
-            # For other element types, navigation is not supported
-            return {
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Cannot navigate deeper from '#{current_path}'",
-            }
-          end
-        end
-
-        # If we've processed all parts without returning an error, path is valid
-        nil
-      end
-
-      # Find a base type in a schema
-      def find_base_type(schema, type_ref)
-        # Skip built-in types
-        return nil if %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
-                         LOGICAL].include?(type_ref.to_s.upcase)
-
-        # Find the referenced type in the schema
-        if type_ref.is_a?(String)
-          find_schema_element(schema, type_ref)
-        elsif type_ref.respond_to?(:id)
-          # It's already a type object
-          type_ref
-        else
-          nil
+          raise Suma::Error, "Error loading schemas: #{e.message}"
         end
       end
 
-      # Write validation results to the output file
       def write_validation_results(output_file_path, output_file_rel,
   unresolved_links, links_by_file)
         total_links = links_by_file.values.sum(&:size)
 
-        # Prepare results for output
         results = []
         results << "Validation complete. Checked #{total_links} links."
 
@@ -427,17 +154,15 @@ module Suma
         else
           results << "❌ Found #{unresolved_links.size} unresolved links:"
           unresolved_links.each do |issue|
-            results << "#{issue[:file]}:#{issue[:line]} - <<express:#{issue[:link]}>> - #{issue[:reason]}"
+            results << "#{issue.file}:#{issue.line} - <<express:#{issue.link}>> - #{issue.reason}"
           end
         end
 
-        # Write results to output file
         begin
           File.write(output_file_path, results.join("\n"))
           puts "Validation results written to #{output_file_rel}"
         rescue StandardError => e
           puts "Error writing to output file: #{e.message}"
-          # Still print results to console as fallback
           puts results
         end
       end

data/lib/suma/cli.rb

@@ -3,6 +3,7 @@
 require "thor"
 require_relative "thor_ext"
 require_relative "cli/validate"
+require_relative "cli/check_svg_quality"
 require "expressir"
 require "expressir/cli"
 
@@ -97,6 +98,44 @@ module Suma
       desc "validate SUBCOMMAND ...ARGS", "Validate express documents"
       subcommand "validate", Cli::Validate
 
+      desc "check_svg_quality [PATH]",
+           "Check SVG quality and sort by severity (critical files first)"
+      option :pattern, type: :string, default: Cli::CheckSvgQuality::DEFAULT_PATTERN,
+                       desc: "Glob pattern for finding SVG files"
+      option :profile, type: :string,
+                       default: Cli::CheckSvgQuality::DEFAULT_PROFILE,
+                       desc: "Validation profile to use (metanorma, svg_1_2_rfc, etc.)"
+      option :format, type: :string, default: "terminal",
+                      desc: "Output format: terminal, yaml, json"
+      option :output, type: :string, aliases: "-o",
+                      desc: "Output file path"
+      option :min_errors, type: :numeric,
+                          desc: "Minimum error count threshold"
+      option :limit, type: :numeric, default: nil,
+                     desc: "Maximum number of files to show (default: unlimited)"
+      option :sort, type: :string, default: "errors",
+                    desc: "Sort by: errors (most errors first) or quality (lowest scores first)"
+      option :progress, type: :boolean, default: false,
+                        desc: "Show progress during processing"
+      option :summary_only, type: :boolean, default: false,
+                            desc: "Show only summary"
+      def check_svg_quality(path = Cli::CheckSvgQuality::DATA_PATH)
+        require_relative "cli/check_svg_quality"
+
+        analyzer = Cli::CheckSvgQuality.new(
+          pattern: options[:pattern],
+          profile: options[:profile],
+          format: options[:format],
+          output: options[:output],
+          min_errors: options[:min_errors],
+          summary_only: options[:summary_only],
+          progress: options[:progress],
+          limit: options[:limit],
+          sort: options[:sort],
+        )
+        analyzer.run(path)
+      end
+
       desc "expressir SUBCOMMAND ...ARGS", "Expressir commands"
       subcommand "expressir", Expressir::Cli
 

data/lib/suma/collection_manifest.rb

@@ -8,7 +8,6 @@ module Suma
     attribute :schemas_only, Lutaml::Model::Type::Boolean
     attribute :entry, CollectionManifest, collection: true,
                                           initialize_empty: true
-    # attribute :schema_source, Lutaml::Model::Type::String
     attr_accessor :schema_config
 
     yaml do
@@ -59,9 +58,9 @@ module Suma
       export_config
     end
 
-    def lookup(attr_sym, match)
-      results = entry.select { |e| e.send(attr_sym) == match }
-      results << self if send(attr_sym) == match
+    def lookup_schemas_only
+      results = entry.select(&:schemas_only)
+      results << self if schemas_only
       results
     end