suma 0.3.0 → 0.5.0

data/lib/suma/link_validator.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 require "expressir"
+require "suma/link_validation"
 
 module Suma
   LinkValidationResult = Struct.new(:file, :line, :link, :reason,
                                     keyword_init: true)
 
   class LinkValidator
+    autoload :Step, "suma/link_validator/step"
+
     def initialize(index)
       @index = index
     end
@@ -28,7 +31,7 @@ module Suma
       content = File.read(file)
       index = {}
       content.lines.each_with_index do |line, idx|
-        line.scan(/<<express:([^,>]+)(?:,[^>]+)?>>/).flatten.each do |link|
+        line.scan(LinkValidation::EXPRESS_LINK_PATTERN).flatten.each do |link|
           index[link] ||= idx
         end
       end
@@ -99,113 +102,34 @@ module Suma
     end
 
     def validate_deep_path(schema, element, path_parts, file, line_idx,
-full_link)
+                           full_link)
       current = element
       current_path = "#{schema.id}.#{element.id}"
+      context = Step::Context.new(file: file, line: line_idx, link: full_link,
+                                  schema: schema, path: current_path)
 
       path_parts.each do |part|
-        case current
-        when Expressir::Model::Declarations::Entity
-          attribute = current.attributes&.find do |a|
-            a.id.downcase == part.downcase
-          end
-
-          unless attribute
-            return LinkValidationResult.new(
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Attribute '#{part}' not found in entity '#{current_path}'",
-            )
-          end
-
-          current = attribute
-          current_path += ".#{part}"
-
-        when Expressir::Model::Declarations::Type
-          underlying = current.underlying_type
-
-          if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
-            enum_value = underlying.items.find do |e|
-              e.id.downcase == part.downcase
-            end
-
-            unless enum_value
-              return LinkValidationResult.new(
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Enumeration value '#{part}' not found in type '#{current_path}'",
-              )
-            end
-
-            current = enum_value
-            current_path += ".#{part}"
-
-          elsif underlying
-            base_type = find_base_type(schema, underlying)
-
-            unless base_type
-              return LinkValidationResult.new(
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Base type not found for '#{current_path}'",
-              )
-            end
-
-            current = base_type
-
-          else
-            return LinkValidationResult.new(
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Cannot navigate deeper from type '#{current_path}'",
-            )
-          end
+        step = Step.for(current)
+        return unsupported_step_failure(context) unless step
 
-        else
-          return LinkValidationResult.new(
-            file: file,
-            line: line_idx + 1,
-            link: full_link,
-            reason: "Cannot navigate deeper from '#{current_path}'",
-          )
-        end
+        outcome = step.new.navigate(current, part, context)
+        return outcome if outcome.is_a?(LinkValidationResult)
+
+        current = outcome
+        current_path += ".#{part}"
+        context.path = current_path
       end
 
       nil
     end
 
-    def find_base_type(schema, type_ref)
-      return nil if %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
-                       LOGICAL].include?(type_ref.to_s.upcase)
-
-      if type_ref.is_a?(String)
-        find_schema_element(schema, type_ref)
-      elsif type_ref.is_a?(Expressir::Model::ModelElement)
-        type_ref
-      end
-    end
-
-    def find_schema_element(schema, element_name)
-      [
-        schema.entities,
-        schema.types,
-        schema.constants,
-        schema.functions,
-        schema.rules,
-        schema.procedures,
-        schema.subtype_constraints,
-      ].each do |collection|
-        element = collection&.find do |e|
-          e.id.downcase == element_name.downcase
-        end
-        return element if element
-      end
-
-      nil
+    def unsupported_step_failure(context)
+      LinkValidationResult.new(
+        file: context.file,
+        line: context.line + 1,
+        link: context.link,
+        reason: "Cannot navigate deeper from '#{context.path}'",
+      )
     end
   end
 end

data/lib/suma/note_processor.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "glossarist"
+
+module Suma
+  # Cleans raw EXPRESS entity remarks into a list of
+  # +Glossarist::V3::DetailedDefinition+ notes.
+  #
+  # Pipeline:
+  #   1. trim to first sentence / first complete list
+  #   2. convert express xrefs to URN mentions
+  #   3. drop redundant "X is a type of {{...}}" opening notes
+  #   4. drop notes containing image references or <<...>> blocks
+  #   5. drop notes that duplicate the first definition
+  #
+  # Pure: no I/O, no schema knowledge. The schema domain label and
+  # URN composition are injected via +xref_to_mention+ so this module
+  # stays independent of +Suma::Urn+ and the surrounding extractor.
+  class NoteProcessor
+    REDUNDANT_NOTE_REGEX =
+      %r{
+        ^An?                   # Starts with "A" or "An"
+        \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
+        (\sa|\san)?            # Optional " a" or " an"
+        \s\{\{[^\}]*\}\}       # Text in double curly braces
+        \s*?\.?$               # Optional whitespace and period at the end
+      }x
+
+    IMAGE_REF_SUBSTRING = "image::"
+    DOUBLE_ANGLE_PATTERN = /<<(.*?){1,999}>>/
+    SEE_TAIL_PATTERN = /\s+\(see(.*?){1,999}\)/
+    XREF_WITH_DISPLAY = /<<express:([\w.]+),([\w. ][\w. ]*)>>/
+    XREF_WITHOUT_DISPLAY = /<<express:([\w.]+)>>/
+    INLINE_COMMENT_PATTERN = /\n\/\/.*?\n/
+    BULLET_LIST_PATTERN = /^\s*[*\-+]\s+/
+    NUMBERED_LIST_PATTERN = /^\s*\d+\.\s+/
+    CONTINUATION_PLUS = /^\+\s*$/
+    CONTINUATION_DASHES = /^--\s*$/
+    INDENTED_PATTERN = /^\s{2,}/
+    SENTENCE_BREAK_NEWLINE = /\.\n/
+    SENTENCE_BREAK_SPACE = /\. /
+
+    attr_reader :remarks, :definitions, :xref_to_mention
+
+    def initialize(remarks, definitions:, xref_to_mention:)
+      @remarks = remarks
+      @definitions = definitions
+      @xref_to_mention = xref_to_mention
+    end
+
+    def self.call(remarks, definitions:, xref_to_mention:)
+      new(remarks, definitions: definitions, xref_to_mention: xref_to_mention).call
+    end
+
+    def call
+      notes = build_initial_notes
+      notes = only_keep_first_sentence(notes)
+      notes = remove_see_content(notes)
+      notes = remove_redundant_note(notes)
+      notes = remove_invalid_references(notes)
+      compare_with_definitions(notes, definitions)
+    end
+
+    private
+
+    def build_initial_notes
+      trimmed = trim_definition(remarks)
+      return [] unless trimmed && !trimmed.empty?
+
+      [Glossarist::V3::DetailedDefinition.new(content: trimmed)]
+    end
+
+    def trim_definition(definition)
+      return nil if definition.nil? || definition.empty?
+
+      definition_str = array_to_paragraphs(definition)
+      return nil if definition_str.empty?
+
+      paragraphs = definition_str.split("\n\n")
+      combine_paragraphs(paragraphs)
+    end
+
+    def array_to_paragraphs(definition)
+      definition.is_a?(Array) ? definition.join("\n\n") : definition.to_s
+    end
+
+    def combine_paragraphs(paragraphs)
+      combined = first_block(paragraphs)
+      combined = "#{combined}\n"
+      combined = combined.gsub(INLINE_COMMENT_PATTERN, "\n")
+      combined.strip!
+      convert_xref(combined)
+    end
+
+    def first_block(paragraphs)
+      first_paragraph = paragraphs.first
+      return apply_first_sentence_logic(first_paragraph) unless list_header?(paragraphs)
+
+      complete_list = extract_complete_list(paragraphs, 1)
+      "#{first_paragraph}\n\n#{complete_list}"
+    end
+
+    def list_header?(paragraphs)
+      paragraphs.length > 1 &&
+        paragraphs.first.end_with?(":") &&
+        starts_with_list?(paragraphs[1])
+    end
+
+    def apply_first_sentence_logic(paragraph)
+      new_content = paragraph
+        .split(SENTENCE_BREAK_NEWLINE).first.strip
+        .split(SENTENCE_BREAK_SPACE).first.strip
+
+      new_content.end_with?(".") ? new_content : "#{new_content}."
+    end
+
+    def extract_complete_list(paragraphs, start_index)
+      return paragraphs[start_index] if start_index >= paragraphs.length
+
+      reduce_list_paragraphs(paragraphs, start_index)
+    end
+
+    def reduce_list_paragraphs(paragraphs, start_index)
+      combined = paragraphs[start_index].dup
+      state = ContinuationState.new(combined.include?("--") && !combined.match?(/--.*--/m))
+
+      ((start_index + 1)...paragraphs.length).each do |i|
+        break unless apply_paragraph_step?(combined, paragraphs[i], state)
+      end
+
+      combined
+    end
+
+    # Returns true if iteration should continue, false to stop.
+    def apply_paragraph_step?(combined, para, state)
+      case classify_paragraph(para, state)
+      when :continuation_toggle
+        state.toggle!
+        append!(combined, para)
+      when :continuation
+        append!(combined, para)
+      when :list_member
+        append!(combined, para)
+        state.toggle! if opens_continuation_block?(para)
+      else
+        return false
+      end
+      true
+    end
+
+    def append!(combined, para)
+      combined << "\n\n#{para}"
+    end
+
+    def opens_continuation_block?(para)
+      para.include?("--") && !para.match?(/--.*--/m)
+    end
+
+    def classify_paragraph(para, state)
+      return :continuation_toggle if para.match?(CONTINUATION_DASHES) || para.end_with?("--")
+      return :continuation if state.in_block?
+      return :list_member if starts_with_list?(para) || list_continuation?(para)
+
+      :stop
+    end
+
+    # Mutable flag tracking whether we are inside a "-- ... --"
+    # continuation block while walking paragraphs.
+    class ContinuationState
+      def initialize(in_block)
+        @in_block = in_block
+      end
+
+      def in_block?
+        @in_block
+      end
+
+      def toggle!
+        @in_block = !@in_block
+      end
+    end
+    private_constant :ContinuationState
+
+    def list_continuation?(content)
+      return false if content.nil? || content.empty?
+
+      content.match?(CONTINUATION_PLUS) ||
+        content.match?(CONTINUATION_DASHES) ||
+        content.match?(INDENTED_PATTERN) ||
+        content.start_with?("which", "where", "that")
+    end
+
+    def starts_with_list?(content)
+      return false if content.nil? || content.empty?
+
+      content.match?(BULLET_LIST_PATTERN) || content.match?(NUMBERED_LIST_PATTERN)
+    end
+
+    def only_keep_first_sentence(notes)
+      notes.each do |note|
+        next unless note&.content
+        next if preserve_complete_structure?(note.content)
+
+        new_content = note.content
+          .split(SENTENCE_BREAK_NEWLINE).first.strip
+          .split(SENTENCE_BREAK_SPACE).first.strip
+        note.content = new_content.end_with?(".") ? new_content : "#{new_content}."
+      end
+    end
+
+    def preserve_complete_structure?(content)
+      return false if content.nil? || content.empty?
+
+      lines = content.split("\n")
+      return false unless first_line_is_header?(lines)
+      return false if first_line_has_period?(lines.first.strip)
+
+      remaining_content = lines[1..].join("\n")
+      starts_with_list?(remaining_content.strip)
+    end
+
+    def first_line_is_header?(lines)
+      lines.first&.strip&.end_with?(":") && lines.length > 1
+    end
+
+    def first_line_has_period?(first_line)
+      first_line.count(".").positive?
+    end
+
+    def remove_see_content(notes)
+      notes.each do |note|
+        note.content = note.content.gsub(SEE_TAIL_PATTERN, "")
+      end
+    end
+
+    def remove_redundant_note(notes)
+      notes.reject do |note|
+        note.content.match?(REDUNDANT_NOTE_REGEX) &&
+          !note.content.include?("\n")
+      end
+    end
+
+    def remove_invalid_references(notes)
+      notes.reject do |note|
+        note.content.include?(IMAGE_REF_SUBSTRING) ||
+          note.content.match?(DOUBLE_ANGLE_PATTERN)
+      end
+    end
+
+    def compare_with_definitions(notes, definitions)
+      return [] if notes&.first&.content == definitions&.first&.content
+
+      notes
+    end
+
+    def convert_xref(content)
+      content
+        .gsub(XREF_WITHOUT_DISPLAY) do |_match|
+          full_ref = Regexp.last_match(1)
+          display = full_ref.split(".").last
+          xref_to_mention.call(full_ref, display)
+        end
+        .gsub(XREF_WITH_DISPLAY) do |_match|
+          full_ref = Regexp.last_match(1)
+          display = Regexp.last_match(2)
+          xref_to_mention.call(full_ref, display)
+        end
+    end
+  end
+end

data/lib/suma/schema_collection.rb

@@ -40,7 +40,7 @@ module Suma
       finalize
 
       exporter = SchemaExporter.new(
-        schemas: @config.schemas,
+        schemas: schemas.values,
         output_path: @output_path_schemas,
         options: { annotations: false },
       )

data/lib/suma/schema_comparer.rb

@@ -91,7 +91,7 @@ module Suma
         existing_schema = Expressir::Changes::SchemaChange.from_file(output_path)
       end
 
-      converter = EengineConverter.new(xml_path, schema_name)
+      converter = EengineConverter.new(schema_name, File.read(xml_path))
       change_schema = converter.convert(
         version: options[:version],
         existing_change_schema: existing_schema,

data/lib/suma/schema_exporter.rb

@@ -3,8 +3,19 @@
 require "fileutils"
 
 module Suma
-  # SchemaExporter exports EXPRESS schemas from a manifest
-  # with configurable options for annotations and ZIP packaging
+  # Exports EXPRESS schemas to a directory, with optional ZIP packaging.
+  #
+  # Pure sink: the exporter accepts already-loaded +Suma::ExpressSchema+
+  # instances and writes their content to disk. Construction of those
+  # instances (with the right +output_path+ and +is_standalone_file+
+  # flags) is the caller's responsibility — the exporter does not
+  # reach across the seam to inspect manifest entries or classify
+  # schema types itself.
+  #
+  # This is a deep module: a small interface (one +export+ method, one
+  # option hash) backed by save_exp + zip packaging. The CLI and
+  # SchemaCollection adapters construct ExpressSchema instances; the
+  # exporter never inspects their shape.
   class SchemaExporter
     attr_reader :schemas, :output_path, :options
 
@@ -35,30 +46,7 @@ module Suma
 
     def export_to_directory(schemas)
       schemas.each do |schema|
-        export_single_schema(schema)
-      end
-    end
-
-    def export_single_schema(schema)
-      is_standalone = !schema.is_a?(Expressir::SchemaManifestEntry)
-      schema_output_path = determine_output_path(schema, is_standalone)
-
-      express_schema = ExpressSchema.new(
-        id: schema.id,
-        path: schema.path.to_s,
-        output_path: schema_output_path,
-        is_standalone_file: is_standalone,
-      )
-
-      express_schema.save_exp(with_annotations: options[:annotations])
-    end
-
-    def determine_output_path(schema, is_standalone)
-      if is_standalone
-        output_path.to_s
-      else
-        category = SchemaCategory.for_schema(id: schema.id, path: schema.path)
-        output_path.join(category.directory).to_s
+        schema.save_exp(with_annotations: options[:annotations])
       end
     end
 

data/lib/suma/svg_quality/scanner.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "svg_conform"
+
+module Suma
+  module SvgQuality
+    # Deep module behind the SVG-quality seam: takes paths in, returns
+    # +Report+ / +BatchReport+ out. Owns validator construction and
+    # per-file result capture. Does not own file discovery, sorting,
+    # filtering, or presentation — those stay in the CLI adapter.
+    #
+    # The progress adapter is injected: pass any object responding to
+    # +#call(index, total, report)+. +NullProgress+ is the default
+    # no-op; the CLI passes a lambda that writes to +$stderr+.
+    class Scanner
+      DEFAULT_PROFILE = :metanorma
+
+      attr_reader :profile, :progress
+
+      def initialize(profile: DEFAULT_PROFILE,
+                     progress: NullProgress.new)
+        @profile = profile
+        @progress = progress
+      end
+
+      def scan(paths)
+        validator = build_validator
+        reports = paths.each_with_index.map do |path, index|
+          scan_one(validator, path, index, paths.size)
+        end
+        BatchReport.new(reports)
+      end
+
+      def scan_file(path)
+        Report.new(path.to_s, build_validator.validate_file(path.to_s,
+                                                            profile: profile))
+      end
+
+      # Default no-op progress adapter. Real progress reporters are
+      # passed by the caller; this satisfies the same interface so the
+      # scanner can be invoked from specs without forcing the
+      # dependency.
+      class NullProgress
+        def call(_index, _total, _report); end
+      end
+
+      private
+
+      def build_validator
+        SvgConform::Validator.new
+      end
+
+      def scan_one(validator, path, index, total)
+        result = validator.validate_file(path.to_s, profile: profile)
+        report = Report.new(path.to_s, result)
+        progress.call(index, total, report)
+        report
+      end
+    end
+  end
+end

data/lib/suma/svg_quality.rb

@@ -6,6 +6,7 @@ module Suma
   module SvgQuality
     autoload :Report,      "suma/svg_quality/report"
     autoload :BatchReport, "suma/svg_quality/batch_report"
+    autoload :Scanner,     "suma/svg_quality/scanner"
 
     module QualityTiers
       CRITICAL = { name: :critical, min_errors: 200, emoji: "💥" }.freeze

data/lib/suma/term_classification.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Suma
+  # Term-extractor-specific classification of an EXPRESS schema.
+  #
+  # Bridges ExpressSchema::Type (the canonical classification shared
+  # across the codebase) and the Glossarist-specific labels
+  # TermExtractor emits: the domain string ("application module" /
+  # "resource") that goes into a concept's +domain+ field, and the
+  # entity-type URN term used in generated concept definitions.
+  #
+  # The mapping is data — a frozen Hash keyed by ExpressSchema::Type
+  # symbol — so adding a new schema type is a one-line addition to
+  # BY_TYPE (open/closed principle). The previous implementation
+  # switched on string keys in three separate places; this consolidates
+  # them into one source of truth.
+  class TermClassification
+    attr_reader :type, :domain_label, :entity_term, :entity_display
+
+    def initialize(type:, domain_label:, entity_term:, entity_display:)
+      @type = type
+      @domain_label = domain_label
+      @entity_term = entity_term
+      @entity_display = entity_display
+      freeze
+    end
+
+    def domain_for(schema_id)
+      "#{domain_label}: #{schema_id}"
+    end
+
+    BY_TYPE = {
+      ExpressSchema::Type::RESOURCE => new(
+        type: ExpressSchema::Type::RESOURCE,
+        domain_label: "resource",
+        entity_term: "express-language.entity_data_type",
+        entity_display: "entity data type",
+      ),
+      ExpressSchema::Type::MODULE_ARM => new(
+        type: ExpressSchema::Type::MODULE_ARM,
+        domain_label: "application module",
+        entity_term: "general.application_object",
+        entity_display: "application object",
+      ),
+      ExpressSchema::Type::MODULE_MIM => new(
+        type: ExpressSchema::Type::MODULE_MIM,
+        domain_label: "application module",
+        entity_term: "express-language.entity_data_type",
+        entity_display: "entity data type",
+      ),
+      ExpressSchema::Type::BUSINESS_OBJECT_MODEL => new(
+        type: ExpressSchema::Type::BUSINESS_OBJECT_MODEL,
+        domain_label: "resource",
+        entity_term: "express-language.entity_data_type",
+        entity_display: "entity data type",
+      ),
+      ExpressSchema::Type::CORE_MODEL => new(
+        type: ExpressSchema::Type::CORE_MODEL,
+        domain_label: "resource",
+        entity_term: "express-language.entity_data_type",
+        entity_display: "entity data type",
+      ),
+      ExpressSchema::Type::STANDALONE => new(
+        type: ExpressSchema::Type::STANDALONE,
+        domain_label: "resource",
+        entity_term: "express-language.entity_data_type",
+        entity_display: "entity data type",
+      ),
+    }.freeze
+
+    def self.for_schema(id:, path:)
+      type = ExpressSchema::Type.classify(id: id, path: path)
+      BY_TYPE.fetch(type) do |t|
+        raise Error, "[suma] no term classification for type #{t.inspect}"
+      end
+    end
+  end
+end