suma 0.3.0 → 0.5.0

data/lib/suma/cli/validate_links.rb

@@ -1,168 +1,24 @@
 # frozen_string_literal: true
 
 require "thor"
-require "expressir"
 
 module Suma
   module Cli
+    # Deprecated: prefer +Cli::Validate#links+ (the +suma validate links+
+    # subcommand). This class is retained as a backwards-compat entry
+    # point — all orchestration now lives in +Suma::LinkValidation+.
     class ValidateLinks < Thor
       desc "extract_and_validate SCHEMAS_FILE DOCUMENTS_PATH [OUTPUT_FILE]",
            "Extract and validate express links without creating intermediate file"
       def extract_and_validate(schemas_file = "schemas-srl.yml",
-                              documents_path = "documents",
-                              output_file = "validation_results.txt")
-        load_dependencies
-        paths = prepare_file_paths(schemas_file, documents_path, output_file)
-
-        schemas_config = load_schemas_config(paths[:schemas_file])
-        exp_files = collect_schema_paths(schemas_config, paths[:schemas_file_rel])
-        adoc_files = find_adoc_files(paths[:documents_path])
-
-        all_files = adoc_files + exp_files
-        display_file_counts(adoc_files, exp_files)
-
-        links_by_file = extract_links(all_files)
-
-        repo = load_express_schemas(schemas_config)
-        index = SchemaIndex.new(repo)
-        unresolved = LinkValidator.new(index).validate(links_by_file)
-
-        write_validation_results(paths[:output_file], paths[:output_file_rel],
-                                 unresolved, links_by_file)
-      end
-
-      private
-
-      def load_dependencies
-        require "expressir"
-        require "ruby-progressbar"
-        require "pathname"
-      end
-
-      def prepare_file_paths(schemas_file, documents_path, output_file)
-        schemas_file_path = Pathname.new(schemas_file).expand_path
-        documents_path_exp = Pathname.new(documents_path).expand_path
-        output_file_path = Pathname.new(output_file).expand_path
-
-        schemas_file_rel = Pathname.new(schemas_file_path).relative_path_from(Pathname.pwd).to_s
-        documents_path_rel = Pathname.new(documents_path_exp).relative_path_from(Pathname.pwd).to_s
-        output_file_rel = Pathname.new(output_file_path).relative_path_from(Pathname.pwd).to_s
-
-        puts "Extracting and validating express links using schemas from #{schemas_file_rel}..."
-        puts "Looking for documents in #{documents_path_rel}..."
-
-        {
-          schemas_file: schemas_file_path,
-          schemas_file_rel: schemas_file_rel,
-          documents_path: documents_path_exp,
-          documents_path_rel: documents_path_rel,
-          output_file: output_file_path,
-          output_file_rel: output_file_rel,
-        }
-      end
-
-      def load_schemas_config(schemas_file_path)
-        schemas_config = Expressir::SchemaManifest.from_yaml(File.read(schemas_file_path))
-        schemas_config.set_initial_path(schemas_file_path.to_s)
-        schemas_config
-      rescue StandardError => e
-        raise Suma::Error, "Error loading schemas file: #{e.message}"
-      end
-
-      def collect_schema_paths(schemas_config, schemas_file_rel)
-        exp_files = schemas_config.schemas.filter_map(&:path)
-        puts "Found #{exp_files.size} EXPRESS schema files from #{schemas_file_rel}"
-        exp_files
-      end
-
-      def find_adoc_files(documents_path)
-        Dir.glob(documents_path.join("**", "*.adoc").to_s)
-      end
-
-      def display_file_counts(adoc_files, exp_files)
-        puts "Found #{adoc_files.size} AsciiDoc files and #{exp_files.size} EXPRESS files"
-      end
-
-      def create_progress_bar(title, total)
-        ProgressBar.create(
-          title: title,
-          total: total,
-          format: "%t: [%B] %p%% %c/%C %e",
-          progress_mark: "=",
-          remainder_mark: " ",
-          length: 80,
-        )
-      end
-
-      def extract_links(files)
-        links_by_file = {}
-        link_count = 0
-
-        progress = create_progress_bar("Processing files", files.size)
-
-        files.each do |file|
-          progress.increment
-          begin
-            content = File.read(file)
-            express_links = content.scan(/<<express:([^,>]+)(?:,[^>]+)?>>/).flatten.uniq
-
-            if express_links.any?
-              links_by_file[file] = express_links
-              link_count += express_links.size
-            end
-          rescue StandardError => e
-            puts "\nWarning: Could not read file #{file}: #{e.message}"
-          end
-        end
-
-        puts "\nExtracted #{link_count} unique express links from #{links_by_file.size} files"
-        links_by_file
-      end
-
-      def load_express_schemas(schemas_config)
-        schema_paths = {}
-        schemas_config.schemas.each { |s| schema_paths[s.id] = s.path }
-
-        puts "Loading #{schema_paths.size} EXPRESS schemas for validation..."
-
-        loading_progress = create_progress_bar("Loading schemas", schema_paths.size)
-
-        begin
-          repo = Expressir::Express::Parser.from_files(schema_paths.values) do |filename, _schemas, error|
-            loading_progress.increment
-            puts "\nWarning: Error loading schema #{filename}: #{error.message}" if error
-          end
-
-          puts "Successfully loaded #{repo.schemas.size} schemas"
-          repo
-        rescue StandardError => e
-          raise Suma::Error, "Error loading schemas: #{e.message}"
-        end
-      end
-
-      def write_validation_results(output_file_path, output_file_rel,
-  unresolved_links, links_by_file)
-        total_links = links_by_file.values.sum(&:size)
-
-        results = []
-        results << "Validation complete. Checked #{total_links} links."
-
-        if unresolved_links.empty?
-          results << "✅ All links resolved successfully!"
-        else
-          results << "❌ Found #{unresolved_links.size} unresolved links:"
-          unresolved_links.each do |issue|
-            results << "#{issue.file}:#{issue.line} - <<express:#{issue.link}>> - #{issue.reason}"
-          end
-        end
-
-        begin
-          File.write(output_file_path, results.join("\n"))
-          puts "Validation results written to #{output_file_rel}"
-        rescue StandardError => e
-          puts "Error writing to output file: #{e.message}"
-          puts results
-        end
+                               documents_path = "documents",
+                               output_file = "validation_results.txt")
+        result = LinkValidation.new(
+          schemas_file: schemas_file,
+          documents_path: documents_path,
+          output_file: output_file,
+        ).call
+        puts LinkValidation.generate_summary(result)
       end
     end
   end

data/lib/suma/eengine_converter.rb

@@ -3,24 +3,25 @@
 require "expressir/commands/changes_import_eengine"
 
 module Suma
-  # Converts eengine comparison XML to Expressir::Changes::SchemaChange
-  # This is a thin wrapper around Expressir's ChangesImportEengine command
+  # Converts eengine comparison XML to Expressir::Changes::SchemaChange.
+  #
+  # Pure converter: takes XML content as a string (no I/O). The caller
+  # is responsible for reading the file. This keeps the object cheap
+  # to construct and easy to test in isolation — no fixture file
+  # required just to instantiate.
   class EengineConverter
-    def initialize(xml_path, schema_name)
-      @xml_path = xml_path
+    def initialize(schema_name, xml_content)
       @schema_name = schema_name
-      @xml_content = File.read(xml_path)
+      @xml_content = xml_content
     end
 
-    # Convert the eengine XML to a ChangeSchema
+    # Convert the eengine XML to a ChangeSchema.
     #
     # @param version [String] Version number for this change version
     # @param existing_change_schema [Expressir::Changes::SchemaChange, nil]
     #   Existing schema to append to, or nil to create new
     # @return [Expressir::Changes::SchemaChange] The updated change schema
     def convert(version:, existing_change_schema: nil)
-      # Use Expressir's built-in conversion which properly handles
-      # HTML elements in descriptions
       Expressir::Commands::ChangesImportEengine.from_xml(
         @xml_content,
         @schema_name,

data/lib/suma/express_reformatter.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Suma
+  # Reformats EXPRESS source into suma's canonical form: comment blocks
+  # (the +(*"...\n*)+ remarks that EXPRESS uses for documentation) are
+  # extracted from their inline positions and appended to the end of
+  # the file, and excess blank lines are collapsed.
+  #
+  # Pure content transformation — no I/O, no Thor, no filesystem. The
+  # CLI adapter handles reading from and writing to disk.
+  #
+  # The transform is idempotent: reformatting an already-reformatted
+  # document returns +changed?: false+.
+  #
+  # Comment extraction uses +String#index+ rather than a single m-mode
+  # regex. The original +/\(\*"(.*?)\n\*\)/m+ is correct functionally
+  # but is polynomial-time on adversarial input (CodeQL
+  # +rb/polynomial-redos+). Scanning for the start marker, then for
+  # the next terminator, is unambiguously linear.
+  module ExpressReformatter
+    Result = Struct.new(:content, :changed?, keyword_init: true) do
+      def content_or_nil
+        changed? ? content : nil
+      end
+    end
+
+    COMMENT_START = '(*"'
+    COMMENT_END = "\n*)"
+    BLANK_RUN_PATTERN = /(\n\n+)/
+    NEWLINE_RUN_PATTERN = /(\n+)/
+
+    module_function
+
+    def call(content)
+      comments = extract_comments(content)
+      return Result.new(content: content, changed?: false) if comments.empty?
+
+      without_comments = strip_comments(content)
+      new_comments = comments.map { |c| "#{COMMENT_START}#{c}#{COMMENT_END}" }
+        .join("\n\n")
+      new_content = "#{without_comments}\n\n#{new_comments}\n"
+      new_content = new_content.gsub(BLANK_RUN_PATTERN, "\n\n")
+
+      changed = normalised_compare(content, new_content) != 0
+      Result.new(content: new_content, changed?: changed)
+    end
+
+    def extract_comments(content)
+      comments = []
+      pos = 0
+      while (start_idx = content.index(COMMENT_START, pos))
+        end_idx = content.index(COMMENT_END, start_idx + COMMENT_START.length)
+        break unless end_idx
+
+        comments << content[(start_idx + COMMENT_START.length)...end_idx]
+        pos = end_idx + COMMENT_END.length
+      end
+      comments
+    end
+
+    def strip_comments(content)
+      return content unless content.index(COMMENT_START, 0)
+
+      stripped = +""
+      last_end = 0
+      each_comment_range(content) do |range|
+        stripped << content[range[:pre_start]...range[:start]]
+        last_end = range[:end_exclusive]
+      end
+      stripped << content[last_end..]
+      stripped.force_encoding(content.encoding)
+    end
+
+    def each_comment_range(content)
+      return enum_for(:each_comment_range, content) unless block_given?
+
+      pos = 0
+      while (start_idx = content.index(COMMENT_START, pos))
+        end_idx = content.index(COMMENT_END, start_idx + COMMENT_START.length)
+        break unless end_idx
+
+        yield pre_start: pos, start: start_idx,
+              end_exclusive: end_idx + COMMENT_END.length
+        pos = end_idx + COMMENT_END.length
+      end
+    end
+
+    def normalised_compare(left, right)
+      left.gsub(NEWLINE_RUN_PATTERN, "\n") <=> right.gsub(NEWLINE_RUN_PATTERN, "\n")
+    end
+
+    private_class_method :normalised_compare
+  end
+end

data/lib/suma/link_validation.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "expressir"
+
+module Suma
+  # Deep module behind the EXPRESS cross-reference validation pipeline.
+  #
+  # Interface: paths in, +Result+ out. The CLI is a thin adapter that
+  # constructs this object and prints the result.
+  #
+  # Owns: loading the schemas manifest, discovering + reading .adoc and
+  # .exp files, extracting express cross-reference links, loading parsed
+  # schemas into a +SchemaIndex+, delegating to +LinkValidator+ for the
+  # actual resolution, and writing the summary file.
+  #
+  # Does not own: presentation (use +LinkValidation.generate_summary+),
+  # command-line argument parsing (the CLI adapter does that), or the
+  # link-resolution rules themselves (that is +LinkValidator+'s job).
+  class LinkValidation
+    EXPRESS_LINK_PATTERN = /<<express:([^,>]+)(?:,[^>]+)?>>/
+
+    attr_reader :schemas_file, :documents_path, :output_file,
+                :progress, :logger
+
+    def initialize(schemas_file:, documents_path:, output_file:,
+                   progress: NullProgress.new, logger: Utils)
+      @schemas_file = Pathname.new(schemas_file).expand_path
+      @documents_path = Pathname.new(documents_path).expand_path
+      @output_file = output_file && Pathname.new(output_file).expand_path
+      @progress = progress
+      @logger = logger
+    end
+
+    def call
+      config = load_schemas_config
+      exp_files = collect_schema_paths(config)
+      adoc_files = find_adoc_files
+      links_by_file = extract_links(adoc_files + exp_files)
+      unresolved = validate_links(config, links_by_file)
+      result = Result.new(
+        adoc_count: adoc_files.size,
+        exp_count: exp_files.size,
+        total_links: links_by_file.values.sum(&:size),
+        unresolved: unresolved,
+      )
+      write_summary(result) if output_file
+      result
+    end
+
+    def self.generate_summary(result)
+      lines = []
+      lines << "Validation complete. Checked #{result.total_links} links."
+      if result.success?
+        lines << "✅ All links resolved successfully!"
+      else
+        lines << "❌ Found #{result.unresolved.size} unresolved links:"
+        result.unresolved.each { |issue| lines << format_issue(issue) }
+      end
+      lines.join("\n")
+    end
+
+    def self.format_issue(issue)
+      "#{issue.file}:#{issue.line} - " \
+        "<<express:#{issue.link}>> - #{issue.reason}"
+    end
+    private_class_method :format_issue
+
+    # Default no-op progress adapter. Callers that want a real progress
+    # bar pass an object responding to +#start(title, total)+ and
+    # +#increment+; this satisfies the same interface without forcing
+    # the dependency.
+    class NullProgress
+      def start(_title, _total); end
+
+      def increment; end
+    end
+
+    Result = Struct.new(
+      :adoc_count,
+      :exp_count,
+      :total_links,
+      :unresolved,
+      keyword_init: true,
+    ) do
+      def success?
+        unresolved.empty?
+      end
+    end
+
+    private
+
+    def load_schemas_config
+      Expressir::SchemaManifest.from_yaml(File.read(schemas_file))
+        .tap { |c| c.set_initial_path(schemas_file.to_s) }
+    rescue StandardError => e
+      raise Error, "Error loading schemas file: #{e.message}"
+    end
+
+    def collect_schema_paths(schemas_config)
+      schemas_config.schemas.filter_map(&:path)
+    end
+
+    def find_adoc_files
+      Dir.glob(documents_path.join("**", "*.adoc").to_s)
+    end
+
+    def extract_links(files)
+      links_by_file = {}
+      progress.start("Processing files", files.size)
+      files.each do |file|
+        links = extract_links_from_file(file)
+        links_by_file[file] = links if links&.any?
+      end
+      links_by_file
+    end
+
+    def extract_links_from_file(file)
+      progress.increment
+      content = File.read(file)
+      content.scan(EXPRESS_LINK_PATTERN).flatten.uniq
+    rescue StandardError => e
+      logger.log "Warning: Could not read file #{file}: #{e.message}"
+      nil
+    end
+
+    def validate_links(schemas_config, links_by_file)
+      paths_by_id = schemas_config.schemas.to_h { |s| [s.id, s.path] }
+      progress.start("Loading schemas", paths_by_id.size)
+      repo = Expressir::Express::Parser.from_files(paths_by_id.values) do |*_args|
+        progress.increment
+      end
+      index = SchemaIndex.new(repo)
+      LinkValidator.new(index).validate(links_by_file)
+    end
+
+    def write_summary(result)
+      FileUtils.mkdir_p(output_file.dirname)
+      File.write(output_file, self.class.generate_summary(result))
+    rescue StandardError => e
+      logger.log "Error writing to output file: #{e.message}"
+    end
+  end
+end

data/lib/suma/link_validator/step.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "expressir"
+
+module Suma
+  class LinkValidator
+    # Per-node-type navigation strategies for +validate_deep_path+.
+    #
+    # Each Step handles one EXPRESS construct (entity, type, ...) and
+    # knows how to walk one path segment from it. The dispatcher in
+    # LinkValidator picks the right Step by +handles?+.
+    #
+    # Adding a new navigable construct means adding a new Step class
+    # and registering it in REGISTRY — no edits to existing Steps or
+    # to validate_deep_path (open/closed principle).
+    module Step
+      Context = Struct.new(:file, :line, :link, :schema, :path,
+                           keyword_init: true)
+
+      # Common failure-result builder. Step subclasses call this to
+      # surface an unresolved link.
+      module Failure
+        def failure(reason, context)
+          LinkValidationResult.new(
+            file: context.file,
+            line: context.line + 1,
+            link: context.link,
+            reason: reason,
+          )
+        end
+      end
+
+      # Navigates one path segment from an Entity by attribute name.
+      class Entity
+        include Failure
+
+        def self.handles?(node)
+          node.is_a?(Expressir::Model::Declarations::Entity)
+        end
+
+        def navigate(node, part, context)
+          attribute = node.attributes&.find { |a| a.id.downcase == part.downcase }
+          unless attribute
+            return failure("Attribute '#{part}' not found in entity '#{context.path}'",
+                           context)
+          end
+
+          attribute
+        end
+      end
+
+      # Navigates one path segment from a Type by either enumeration
+      # value (for enum types) or by re-targeting to the underlying
+      # named type (for non-enum types).
+      class Type
+        include Failure
+
+        PRIMITIVE_TYPE_NAMES = %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
+                                  LOGICAL].freeze
+
+        def self.handles?(node)
+          node.is_a?(Expressir::Model::Declarations::Type)
+        end
+
+        def navigate(node, part, context)
+          underlying = node.underlying_type
+          unless underlying
+            return failure("Cannot navigate deeper from type '#{context.path}'",
+                           context)
+          end
+
+          if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
+            navigate_enum(underlying, part, context)
+          else
+            navigate_underlying(underlying, context)
+          end
+        end
+
+        private
+
+        def navigate_enum(underlying, part, context)
+          enum_value = underlying.items.find { |e| e.id.downcase == part.downcase }
+          unless enum_value
+            return failure("Enumeration value '#{part}' not found in type '#{context.path}'",
+                           context)
+          end
+
+          enum_value
+        end
+
+        def navigate_underlying(underlying, context)
+          base_type = resolve_base_type(context.schema, underlying)
+          unless base_type
+            return failure("Base type not found for '#{context.path}'",
+                           context)
+          end
+
+          base_type
+        end
+
+        def resolve_base_type(schema, type_ref)
+          return nil if PRIMITIVE_TYPE_NAMES.include?(type_ref.to_s.upcase)
+          return find_schema_element(schema, type_ref) if type_ref.is_a?(String)
+
+          type_ref if type_ref.is_a?(Expressir::Model::ModelElement)
+        end
+
+        def find_schema_element(schema, name)
+          SchemaElementLookup.find(schema, name)
+        end
+      end
+
+      # Lookup helper shared across Step classes that need to resolve
+      # a named schema element (used by Type for underlying-type
+      # resolution). Lives behind its own module so Steps don't reach
+      # into LinkValidator's private methods.
+      module SchemaElementLookup
+        def self.find(schema, name)
+          collections(schema).each do |collection|
+            element = collection&.find { |e| e.id.downcase == name.downcase }
+            return element if element
+          end
+          nil
+        end
+
+        def self.collections(schema)
+          [
+            schema.entities,
+            schema.types,
+            schema.constants,
+            schema.functions,
+            schema.rules,
+            schema.procedures,
+            schema.subtype_constraints,
+          ]
+        end
+        private_class_method :collections
+      end
+
+      REGISTRY = [Entity, Type].freeze
+
+      # Returns the Step class that handles +node+, or +nil+ if no
+      # registered Step matches.
+      def self.for(node)
+        REGISTRY.find { |step| step.handles?(node) }
+      end
+    end
+  end
+end