suma 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 304c4c7a2ec03c83604a127370259b022e0102d303c4951c2d730d3681d3ad39
-  data.tar.gz: b8d9bcd807ea0aadb4f038dd58823d829d495827e4785316cb7c715e79b512cb
+  metadata.gz: 165792c4ce4a7a5a23cf0c8fdc649882f68235e65fcc6890a0d11ee5997c063a
+  data.tar.gz: 05fdff4f378207ba636169838b4d3631fcfd30009f254ab5eed85a191c12ec5a
 SHA512:
-  metadata.gz: 21977b339cd0a16936940c50de766ba2130f3dc46df7965db1de04360a65c1d685e1b59786e9a49548d0e79f1d20befcf732a66482b6e8b8793a547bd557d28e
-  data.tar.gz: bd1122e6d73a9a22b3a3830c7cf654c909a541e7c332808349b15bb469f772338ad7a84a25590f1c8d13b46b84d8399673fa2f43f16f5d943e0e1f8e2469a08a
+  metadata.gz: 227dd455539080a5b257f1d5cbb563c5a2fe78856d7eaac4035402060f9415dcbb80ec87928a971a77eccf8585e23c0f7d9e9fd866a8a2cea4471bb098f40ed5
+  data.tar.gz: 47b7edf861d91dc264acd794d6ffdeb1c952aa83da822da3816e450429f81104088d3af68322245a9c70ecf920bcda1762c916e049f0c16c67208f6035960b8f

data/lib/suma/cli/core.rb

@@ -40,14 +40,42 @@ module Suma
       desc "extract-terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
            "Extract terms from SCHEMA_MANIFEST_FILE into " \
            "Glossarist v3 format"
-      def extract_terms(_schema_manifest_file, _glossarist_output_path)
-        Cli::ExtractTerms.start
+      option :urn, type: :string, required: true, aliases: "-u",
+                   desc: "URN for the dataset source " \
+                         "(used for section references)"
+      option :language_code, type: :string, default: "eng", aliases: "-l",
+                             desc: "Language code for the Glossarist"
+      def extract_terms(schema_manifest_file, glossarist_output_path)
+        TermExtractor.new(
+          schema_manifest_file,
+          glossarist_output_path,
+          urn: options[:urn],
+          language_code: options[:language_code],
+        ).call
       end
 
       desc "generate-register SCHEMA_MANIFEST_FILE OUTPUT_PATH",
            "Generate a Glossarist register.yaml with hierarchical sections"
-      def generate_register(_schema_manifest_file, _output_path)
-        Cli::GenerateRegister.start
+      option :urn, type: :string, required: true, aliases: "-u",
+                   desc: "URN prefix for the dataset"
+      option :id, type: :string, required: true,
+                  desc: "Dataset identifier (e.g. iso10303-2-express)"
+      option :ref, type: :string, required: true,
+                   desc: "Human-readable reference label"
+      option :language_code, type: :string, default: "eng", aliases: "-l",
+                             desc: "Language code for section names"
+      option :owner, type: :string, default: Suma::RegisterManifestGenerator::DEFAULT_OWNER,
+                     desc: "Owner of the dataset (e.g. 'ISO/TC 184/SC 4')"
+      def generate_register(schema_manifest_file, output_path)
+        RegisterManifestGenerator.new(
+          schema_manifest_file,
+          output_path,
+          urn: options[:urn],
+          id: options[:id],
+          ref: options[:ref],
+          language_code: options[:language_code],
+          owner: options[:owner],
+        ).generate
       end
 
       desc "convert-jsdai XML_FILE IMAGE_FILE OUTPUT_DIR",

data/lib/suma/eengine_converter.rb

@@ -3,24 +3,25 @@
 require "expressir/commands/changes_import_eengine"
 
 module Suma
-  # Converts eengine comparison XML to Expressir::Changes::SchemaChange
-  # This is a thin wrapper around Expressir's ChangesImportEengine command
+  # Converts eengine comparison XML to Expressir::Changes::SchemaChange.
+  #
+  # Pure converter: takes XML content as a string (no I/O). The caller
+  # is responsible for reading the file. This keeps the object cheap
+  # to construct and easy to test in isolation — no fixture file
+  # required just to instantiate.
   class EengineConverter
-    def initialize(xml_path, schema_name)
-      @xml_path = xml_path
+    def initialize(schema_name, xml_content)
       @schema_name = schema_name
-      @xml_content = File.read(xml_path)
+      @xml_content = xml_content
     end
 
-    # Convert the eengine XML to a ChangeSchema
+    # Convert the eengine XML to a ChangeSchema.
     #
     # @param version [String] Version number for this change version
     # @param existing_change_schema [Expressir::Changes::SchemaChange, nil]
     #   Existing schema to append to, or nil to create new
     # @return [Expressir::Changes::SchemaChange] The updated change schema
     def convert(version:, existing_change_schema: nil)
-      # Use Expressir's built-in conversion which properly handles
-      # HTML elements in descriptions
       Expressir::Commands::ChangesImportEengine.from_xml(
         @xml_content,
         @schema_name,

data/lib/suma/link_validator/step.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "expressir"
+
+module Suma
+  class LinkValidator
+    # Per-node-type navigation strategies for +validate_deep_path+.
+    #
+    # Each Step handles one EXPRESS construct (entity, type, ...) and
+    # knows how to walk one path segment from it. The dispatcher in
+    # LinkValidator picks the right Step by +handles?+.
+    #
+    # Adding a new navigable construct means adding a new Step class
+    # and registering it in REGISTRY — no edits to existing Steps or
+    # to validate_deep_path (open/closed principle).
+    module Step
+      Context = Struct.new(:file, :line, :link, :schema, :path,
+                           keyword_init: true)
+
+      # Common failure-result builder. Step subclasses call this to
+      # surface an unresolved link.
+      module Failure
+        def failure(reason, context)
+          LinkValidationResult.new(
+            file: context.file,
+            line: context.line + 1,
+            link: context.link,
+            reason: reason,
+          )
+        end
+      end
+
+      # Navigates one path segment from an Entity by attribute name.
+      class Entity
+        include Failure
+
+        def self.handles?(node)
+          node.is_a?(Expressir::Model::Declarations::Entity)
+        end
+
+        def navigate(node, part, context)
+          attribute = node.attributes&.find { |a| a.id.downcase == part.downcase }
+          unless attribute
+            return failure("Attribute '#{part}' not found in entity '#{context.path}'",
+                           context)
+          end
+
+          attribute
+        end
+      end
+
+      # Navigates one path segment from a Type by either enumeration
+      # value (for enum types) or by re-targeting to the underlying
+      # named type (for non-enum types).
+      class Type
+        include Failure
+
+        PRIMITIVE_TYPE_NAMES = %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
+                                  LOGICAL].freeze
+
+        def self.handles?(node)
+          node.is_a?(Expressir::Model::Declarations::Type)
+        end
+
+        def navigate(node, part, context)
+          underlying = node.underlying_type
+          unless underlying
+            return failure("Cannot navigate deeper from type '#{context.path}'",
+                           context)
+          end
+
+          if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
+            navigate_enum(underlying, part, context)
+          else
+            navigate_underlying(underlying, context)
+          end
+        end
+
+        private
+
+        def navigate_enum(underlying, part, context)
+          enum_value = underlying.items.find { |e| e.id.downcase == part.downcase }
+          unless enum_value
+            return failure("Enumeration value '#{part}' not found in type '#{context.path}'",
+                           context)
+          end
+
+          enum_value
+        end
+
+        def navigate_underlying(underlying, context)
+          base_type = resolve_base_type(context.schema, underlying)
+          unless base_type
+            return failure("Base type not found for '#{context.path}'",
+                           context)
+          end
+
+          base_type
+        end
+
+        def resolve_base_type(schema, type_ref)
+          return nil if PRIMITIVE_TYPE_NAMES.include?(type_ref.to_s.upcase)
+          return find_schema_element(schema, type_ref) if type_ref.is_a?(String)
+
+          type_ref if type_ref.is_a?(Expressir::Model::ModelElement)
+        end
+
+        def find_schema_element(schema, name)
+          SchemaElementLookup.find(schema, name)
+        end
+      end
+
+      # Lookup helper shared across Step classes that need to resolve
+      # a named schema element (used by Type for underlying-type
+      # resolution). Lives behind its own module so Steps don't reach
+      # into LinkValidator's private methods.
+      module SchemaElementLookup
+        def self.find(schema, name)
+          collections(schema).each do |collection|
+            element = collection&.find { |e| e.id.downcase == name.downcase }
+            return element if element
+          end
+          nil
+        end
+
+        def self.collections(schema)
+          [
+            schema.entities,
+            schema.types,
+            schema.constants,
+            schema.functions,
+            schema.rules,
+            schema.procedures,
+            schema.subtype_constraints,
+          ]
+        end
+        private_class_method :collections
+      end
+
+      REGISTRY = [Entity, Type].freeze
+
+      # Returns the Step class that handles +node+, or +nil+ if no
+      # registered Step matches.
+      def self.for(node)
+        REGISTRY.find { |step| step.handles?(node) }
+      end
+    end
+  end
+end

data/lib/suma/link_validator.rb

@@ -8,6 +8,8 @@ module Suma
                                     keyword_init: true)
 
   class LinkValidator
+    autoload :Step, "suma/link_validator/step"
+
     def initialize(index)
       @index = index
     end
@@ -100,113 +102,34 @@ module Suma
     end
 
     def validate_deep_path(schema, element, path_parts, file, line_idx,
-full_link)
+                           full_link)
       current = element
       current_path = "#{schema.id}.#{element.id}"
+      context = Step::Context.new(file: file, line: line_idx, link: full_link,
+                                  schema: schema, path: current_path)
 
       path_parts.each do |part|
-        case current
-        when Expressir::Model::Declarations::Entity
-          attribute = current.attributes&.find do |a|
-            a.id.downcase == part.downcase
-          end
-
-          unless attribute
-            return LinkValidationResult.new(
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Attribute '#{part}' not found in entity '#{current_path}'",
-            )
-          end
-
-          current = attribute
-          current_path += ".#{part}"
-
-        when Expressir::Model::Declarations::Type
-          underlying = current.underlying_type
-
-          if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
-            enum_value = underlying.items.find do |e|
-              e.id.downcase == part.downcase
-            end
-
-            unless enum_value
-              return LinkValidationResult.new(
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Enumeration value '#{part}' not found in type '#{current_path}'",
-              )
-            end
-
-            current = enum_value
-            current_path += ".#{part}"
-
-          elsif underlying
-            base_type = find_base_type(schema, underlying)
-
-            unless base_type
-              return LinkValidationResult.new(
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Base type not found for '#{current_path}'",
-              )
-            end
-
-            current = base_type
-
-          else
-            return LinkValidationResult.new(
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Cannot navigate deeper from type '#{current_path}'",
-            )
-          end
+        step = Step.for(current)
+        return unsupported_step_failure(context) unless step
 
-        else
-          return LinkValidationResult.new(
-            file: file,
-            line: line_idx + 1,
-            link: full_link,
-            reason: "Cannot navigate deeper from '#{current_path}'",
-          )
-        end
+        outcome = step.new.navigate(current, part, context)
+        return outcome if outcome.is_a?(LinkValidationResult)
+
+        current = outcome
+        current_path += ".#{part}"
+        context.path = current_path
       end
 
       nil
     end
 
-    def find_base_type(schema, type_ref)
-      return nil if %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
-                       LOGICAL].include?(type_ref.to_s.upcase)
-
-      if type_ref.is_a?(String)
-        find_schema_element(schema, type_ref)
-      elsif type_ref.is_a?(Expressir::Model::ModelElement)
-        type_ref
-      end
-    end
-
-    def find_schema_element(schema, element_name)
-      [
-        schema.entities,
-        schema.types,
-        schema.constants,
-        schema.functions,
-        schema.rules,
-        schema.procedures,
-        schema.subtype_constraints,
-      ].each do |collection|
-        element = collection&.find do |e|
-          e.id.downcase == element_name.downcase
-        end
-        return element if element
-      end
-
-      nil
+    def unsupported_step_failure(context)
+      LinkValidationResult.new(
+        file: context.file,
+        line: context.line + 1,
+        link: context.link,
+        reason: "Cannot navigate deeper from '#{context.path}'",
+      )
     end
   end
 end

data/lib/suma/note_processor.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "glossarist"
+
+module Suma
+  # Cleans raw EXPRESS entity remarks into a list of
+  # +Glossarist::V3::DetailedDefinition+ notes.
+  #
+  # Pipeline:
+  #   1. trim to first sentence / first complete list
+  #   2. convert express xrefs to URN mentions
+  #   3. drop redundant "X is a type of {{...}}" opening notes
+  #   4. drop notes containing image references or <<...>> blocks
+  #   5. drop notes that duplicate the first definition
+  #
+  # Pure: no I/O, no schema knowledge. The schema domain label and
+  # URN composition are injected via +xref_to_mention+ so this module
+  # stays independent of +Suma::Urn+ and the surrounding extractor.
+  class NoteProcessor
+    REDUNDANT_NOTE_REGEX =
+      %r{
+        ^An?                   # Starts with "A" or "An"
+        \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
+        (\sa|\san)?            # Optional " a" or " an"
+        \s\{\{[^\}]*\}\}       # Text in double curly braces
+        \s*?\.?$               # Optional whitespace and period at the end
+      }x
+
+    IMAGE_REF_SUBSTRING = "image::"
+    DOUBLE_ANGLE_PATTERN = /<<(.*?){1,999}>>/
+    SEE_TAIL_PATTERN = /\s+\(see(.*?){1,999}\)/
+    XREF_WITH_DISPLAY = /<<express:([\w.]+),([\w. ][\w. ]*)>>/
+    XREF_WITHOUT_DISPLAY = /<<express:([\w.]+)>>/
+    INLINE_COMMENT_PATTERN = /\n\/\/.*?\n/
+    BULLET_LIST_PATTERN = /^\s*[*\-+]\s+/
+    NUMBERED_LIST_PATTERN = /^\s*\d+\.\s+/
+    CONTINUATION_PLUS = /^\+\s*$/
+    CONTINUATION_DASHES = /^--\s*$/
+    INDENTED_PATTERN = /^\s{2,}/
+    SENTENCE_BREAK_NEWLINE = /\.\n/
+    SENTENCE_BREAK_SPACE = /\. /
+
+    attr_reader :remarks, :definitions, :xref_to_mention
+
+    def initialize(remarks, definitions:, xref_to_mention:)
+      @remarks = remarks
+      @definitions = definitions
+      @xref_to_mention = xref_to_mention
+    end
+
+    def self.call(remarks, definitions:, xref_to_mention:)
+      new(remarks, definitions: definitions, xref_to_mention: xref_to_mention).call
+    end
+
+    def call
+      notes = build_initial_notes
+      notes = only_keep_first_sentence(notes)
+      notes = remove_see_content(notes)
+      notes = remove_redundant_note(notes)
+      notes = remove_invalid_references(notes)
+      compare_with_definitions(notes, definitions)
+    end
+
+    private
+
+    def build_initial_notes
+      trimmed = trim_definition(remarks)
+      return [] unless trimmed && !trimmed.empty?
+
+      [Glossarist::V3::DetailedDefinition.new(content: trimmed)]
+    end
+
+    def trim_definition(definition)
+      return nil if definition.nil? || definition.empty?
+
+      definition_str = array_to_paragraphs(definition)
+      return nil if definition_str.empty?
+
+      paragraphs = definition_str.split("\n\n")
+      combine_paragraphs(paragraphs)
+    end
+
+    def array_to_paragraphs(definition)
+      definition.is_a?(Array) ? definition.join("\n\n") : definition.to_s
+    end
+
+    def combine_paragraphs(paragraphs)
+      combined = first_block(paragraphs)
+      combined = "#{combined}\n"
+      combined = combined.gsub(INLINE_COMMENT_PATTERN, "\n")
+      combined.strip!
+      convert_xref(combined)
+    end
+
+    def first_block(paragraphs)
+      first_paragraph = paragraphs.first
+      return apply_first_sentence_logic(first_paragraph) unless list_header?(paragraphs)
+
+      complete_list = extract_complete_list(paragraphs, 1)
+      "#{first_paragraph}\n\n#{complete_list}"
+    end
+
+    def list_header?(paragraphs)
+      paragraphs.length > 1 &&
+        paragraphs.first.end_with?(":") &&
+        starts_with_list?(paragraphs[1])
+    end
+
+    def apply_first_sentence_logic(paragraph)
+      new_content = paragraph
+        .split(SENTENCE_BREAK_NEWLINE).first.strip
+        .split(SENTENCE_BREAK_SPACE).first.strip
+
+      new_content.end_with?(".") ? new_content : "#{new_content}."
+    end
+
+    def extract_complete_list(paragraphs, start_index)
+      return paragraphs[start_index] if start_index >= paragraphs.length
+
+      reduce_list_paragraphs(paragraphs, start_index)
+    end
+
+    def reduce_list_paragraphs(paragraphs, start_index)
+      combined = paragraphs[start_index].dup
+      state = ContinuationState.new(combined.include?("--") && !combined.match?(/--.*--/m))
+
+      ((start_index + 1)...paragraphs.length).each do |i|
+        break unless apply_paragraph_step?(combined, paragraphs[i], state)
+      end
+
+      combined
+    end
+
+    # Returns true if iteration should continue, false to stop.
+    def apply_paragraph_step?(combined, para, state)
+      case classify_paragraph(para, state)
+      when :continuation_toggle
+        state.toggle!
+        append!(combined, para)
+      when :continuation
+        append!(combined, para)
+      when :list_member
+        append!(combined, para)
+        state.toggle! if opens_continuation_block?(para)
+      else
+        return false
+      end
+      true
+    end
+
+    def append!(combined, para)
+      combined << "\n\n#{para}"
+    end
+
+    def opens_continuation_block?(para)
+      para.include?("--") && !para.match?(/--.*--/m)
+    end
+
+    def classify_paragraph(para, state)
+      return :continuation_toggle if para.match?(CONTINUATION_DASHES) || para.end_with?("--")
+      return :continuation if state.in_block?
+      return :list_member if starts_with_list?(para) || list_continuation?(para)
+
+      :stop
+    end
+
+    # Mutable flag tracking whether we are inside a "-- ... --"
+    # continuation block while walking paragraphs.
+    class ContinuationState
+      def initialize(in_block)
+        @in_block = in_block
+      end
+
+      def in_block?
+        @in_block
+      end
+
+      def toggle!
+        @in_block = !@in_block
+      end
+    end
+    private_constant :ContinuationState
+
+    def list_continuation?(content)
+      return false if content.nil? || content.empty?
+
+      content.match?(CONTINUATION_PLUS) ||
+        content.match?(CONTINUATION_DASHES) ||
+        content.match?(INDENTED_PATTERN) ||
+        content.start_with?("which", "where", "that")
+    end
+
+    def starts_with_list?(content)
+      return false if content.nil? || content.empty?
+
+      content.match?(BULLET_LIST_PATTERN) || content.match?(NUMBERED_LIST_PATTERN)
+    end
+
+    def only_keep_first_sentence(notes)
+      notes.each do |note|
+        next unless note&.content
+        next if preserve_complete_structure?(note.content)
+
+        new_content = note.content
+          .split(SENTENCE_BREAK_NEWLINE).first.strip
+          .split(SENTENCE_BREAK_SPACE).first.strip
+        note.content = new_content.end_with?(".") ? new_content : "#{new_content}."
+      end
+    end
+
+    def preserve_complete_structure?(content)
+      return false if content.nil? || content.empty?
+
+      lines = content.split("\n")
+      return false unless first_line_is_header?(lines)
+      return false if first_line_has_period?(lines.first.strip)
+
+      remaining_content = lines[1..].join("\n")
+      starts_with_list?(remaining_content.strip)
+    end
+
+    def first_line_is_header?(lines)
+      lines.first&.strip&.end_with?(":") && lines.length > 1
+    end
+
+    def first_line_has_period?(first_line)
+      first_line.count(".").positive?
+    end
+
+    def remove_see_content(notes)
+      notes.each do |note|
+        note.content = note.content.gsub(SEE_TAIL_PATTERN, "")
+      end
+    end
+
+    def remove_redundant_note(notes)
+      notes.reject do |note|
+        note.content.match?(REDUNDANT_NOTE_REGEX) &&
+          !note.content.include?("\n")
+      end
+    end
+
+    def remove_invalid_references(notes)
+      notes.reject do |note|
+        note.content.include?(IMAGE_REF_SUBSTRING) ||
+          note.content.match?(DOUBLE_ANGLE_PATTERN)
+      end
+    end
+
+    def compare_with_definitions(notes, definitions)
+      return [] if notes&.first&.content == definitions&.first&.content
+
+      notes
+    end
+
+    def convert_xref(content)
+      content
+        .gsub(XREF_WITHOUT_DISPLAY) do |_match|
+          full_ref = Regexp.last_match(1)
+          display = full_ref.split(".").last
+          xref_to_mention.call(full_ref, display)
+        end
+        .gsub(XREF_WITH_DISPLAY) do |_match|
+          full_ref = Regexp.last_match(1)
+          display = Regexp.last_match(2)
+          xref_to_mention.call(full_ref, display)
+        end
+    end
+  end
+end

data/lib/suma/schema_comparer.rb

@@ -91,7 +91,7 @@ module Suma
         existing_schema = Expressir::Changes::SchemaChange.from_file(output_path)
       end
 
-      converter = EengineConverter.new(xml_path, schema_name)
+      converter = EengineConverter.new(schema_name, File.read(xml_path))
       change_schema = converter.convert(
         version: options[:version],
         existing_change_schema: existing_schema,

data/lib/suma/term_extractor.rb

@@ -6,15 +6,6 @@ require "glossarist"
 
 module Suma
   class TermExtractor
-    REDUNDANT_NOTE_REGEX =
-      %r{
-        ^An?                   # Starts with "A" or "An"
-        \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
-        (\sa|\san)?            # Optional " a" or " an"
-        \s\{\{[^\}]*\}\}       # Text in double curly braces
-        \s*?\.?$               # Optional whitespace and period at the end
-      }x
-
     def initialize(schema_manifest_file, output_path, urn:,
                    language_code: "eng")
       @schema_manifest_file = File.expand_path(schema_manifest_file)
@@ -140,7 +131,11 @@ module Suma
         data.domain = schema_domain
         data.sources = [source_ref] if source_ref
 
-        notes = get_entity_notes(entity, schema_domain, data.definition)
+        notes = NoteProcessor.call(
+          entity.remarks,
+          definitions: data.definition,
+          xref_to_mention: method(:xref_to_mention),
+        )
         data.notes = notes if notes && !notes.empty?
         data.examples = []
       end
@@ -166,6 +161,10 @@ module Suma
       "{{#{urn},#{display}}}"
     end
 
+    def xref_to_mention(full_ref, display)
+      urn_mention(express_entity_urn(full_ref), display)
+    end
+
     def get_section_ref(schema)
       return nil unless @urn
 
@@ -219,183 +218,6 @@ module Suma
       [Glossarist::V3::DetailedDefinition.new(content: definition)]
     end
 
-    def get_entity_notes(entity, schema_domain, definitions)
-      notes = []
-
-      if entity.remarks && !entity.remarks.empty?
-        trimmed_def = trim_definition(entity.remarks)
-        if trimmed_def && !trimmed_def.empty?
-          notes << Glossarist::V3::DetailedDefinition.new(
-            content: convert_express_xref(trimmed_def, schema_domain),
-          )
-        end
-      end
-
-      notes = only_keep_first_sentence(notes)
-      notes = remove_see_content(notes)
-      notes = remove_redundant_note(notes)
-      notes = remove_invalid_references(notes)
-      compare_with_definitions(notes, definitions)
-    end
-
-    def only_keep_first_sentence(notes)
-      notes.each do |note|
-        if note&.content && should_preserve_complete_structure?(note.content)
-          next
-        end
-
-        if note&.content
-          new_content = note.content
-            .split(".\n").first.strip
-            .split(". ").first.strip
-          note.content = new_content.end_with?(".") ? new_content : "#{new_content}."
-        end
-      end
-    end
-
-    def should_preserve_complete_structure?(content)
-      return false if content.nil? || content.empty?
-
-      lines = content.split("\n")
-      first_paragraph = lines.first&.strip
-
-      if first_paragraph&.end_with?(":") && lines.length > 1
-        if first_paragraph.count(".").positive?
-          return false
-        end
-
-        remaining_content = lines[1..].join("\n")
-        return starts_with_list?(remaining_content.strip)
-      end
-
-      false
-    end
-
-    def compare_with_definitions(notes, definitions)
-      if notes&.first&.content == definitions&.first&.content
-        return []
-      end
-
-      notes
-    end
-
-    def remove_invalid_references(notes)
-      notes.reject do |note|
-        note.content.include?("image::") ||
-          note.content.match?(/<<(.*?){1,999}>>/)
-      end
-    end
-
-    def remove_redundant_note(notes)
-      notes.reject do |note|
-        note.content.match?(REDUNDANT_NOTE_REGEX) &&
-          !note.content.include?("\n")
-      end
-    end
-
-    def remove_see_content(notes)
-      notes.each do |note|
-        note.content = note.content.gsub(/\s+\(see(.*?){1,999}\)/, "")
-      end
-    end
-
-    def starts_with_list?(content)
-      return false if content.nil? || content.empty?
-
-      content.match?(/^\s*[*\-+]\s+/) || content.match?(/^\s*\d+\.\s+/)
-    end
-
-    def trim_definition(definition)
-      return nil if definition.nil? || definition.empty?
-
-      definition_str = definition.is_a?(Array) ? definition.join("\n\n") : definition.to_s
-
-      return nil if definition_str.empty?
-
-      paragraphs = definition_str.split("\n\n")
-      first_paragraph = paragraphs.first
-
-      combined = if paragraphs.length == 1
-                   apply_first_sentence_logic(first_paragraph)
-                 elsif first_paragraph.end_with?(":") && paragraphs.length > 1 && starts_with_list?(paragraphs[1])
-                   complete_list = extract_complete_list(paragraphs, 1)
-                   "#{first_paragraph}\n\n#{complete_list}"
-                 else
-                   apply_first_sentence_logic(first_paragraph)
-                 end
-
-      combined = "#{combined}\n"
-      combined.gsub!(/\n\/\/.*?\n/, "\n")
-      combined.strip!
-
-      express_reference_to_mention(combined)
-    end
-
-    def apply_first_sentence_logic(paragraph)
-      new_content = paragraph
-        .split(".\n").first.strip
-        .split(". ").first.strip
-
-      new_content.end_with?(".") ? new_content : "#{new_content}."
-    end
-
-    def extract_complete_list(paragraphs, start_index)
-      return paragraphs[start_index] if start_index >= paragraphs.length
-
-      combined = paragraphs[start_index].dup
-      current_index = start_index + 1
-      in_continuation_block = combined.include?("--") && !combined.match?(/--.*--/m)
-
-      while current_index < paragraphs.length
-        next_para = paragraphs[current_index]
-
-        if next_para.match?(/^--\s*$/) || next_para.end_with?("--")
-          in_continuation_block = !in_continuation_block
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          next
-        end
-
-        if in_continuation_block
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          next
-        end
-
-        if starts_with_list?(next_para) || is_list_continuation?(next_para)
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          in_continuation_block = true if next_para.include?("--") && !next_para.match?(/--.*--/m)
-        else
-          break
-        end
-      end
-
-      combined
-    end
-
-    def is_list_continuation?(content)
-      return false if content.nil? || content.empty?
-
-      content.match?(/^\+\s*$/) ||
-        content.match?(/^--\s*$/) ||
-        content.match?(/^\s{2,}/) ||
-        content.start_with?("which", "where", "that")
-    end
-
-    def express_reference_to_mention(description)
-      description
-        .gsub(/<<express:([\w.]+)>>/) do |_match|
-          full_ref = Regexp.last_match[1]
-          entity_id = full_ref.split(".").last
-          urn_mention(express_entity_urn(full_ref), entity_id)
-        end.gsub(/<<express:([\w.]+),([\w. ][\w. ]*)>>/) do |_match|
-          full_ref = Regexp.last_match[1]
-          display = Regexp.last_match(2)
-          urn_mention(express_entity_urn(full_ref), display)
-        end
-    end
-
     def entity_name_to_text(entity_id)
       entity_id.downcase.gsub("_", " ")
     end
@@ -425,13 +247,5 @@ module Suma
           "#{entity_name_to_text(entity.id)} #{entity_ref}"
       end
     end
-
-    def convert_express_xref(content, _schema_domain)
-      content.gsub(/<<express:([\w.]+),([\w. ][\w. ]*)>>/) do
-        full_ref = Regexp.last_match(1)
-        display = Regexp.last_match(2)
-        urn_mention(express_entity_urn(full_ref), display)
-      end
-    end
   end
 end

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/suma.rb

@@ -16,6 +16,7 @@ module Suma
   autoload :LinkValidation,         "suma/link_validation"
   autoload :LinkValidator,          "suma/link_validator"
   autoload :ManifestTraverser,      "suma/manifest_traverser"
+  autoload :NoteProcessor,          "suma/note_processor"
   autoload :RegisterManifestGenerator, "suma/register_manifest_generator"
   autoload :SchemaCategory,         "suma/schema_category"
   autoload :SchemaCollection,       "suma/schema_collection"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Ribose Inc.
@@ -214,7 +214,9 @@ files:
 - lib/suma/jsdai/figure_xml.rb
 - lib/suma/link_validation.rb
 - lib/suma/link_validator.rb
+- lib/suma/link_validator/step.rb
 - lib/suma/manifest_traverser.rb
+- lib/suma/note_processor.rb
 - lib/suma/processor.rb
 - lib/suma/register_manifest_generator.rb
 - lib/suma/schema_category.rb