suma 0.3.0 → 0.5.0

data/lib/suma/term_extractor.rb

@@ -6,15 +6,6 @@ require "glossarist"
 
 module Suma
   class TermExtractor
-    REDUNDANT_NOTE_REGEX =
-      %r{
-        ^An?                   # Starts with "A" or "An"
-        \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
-        (\sa|\san)?            # Optional " a" or " an"
-        \s\{\{[^\}]*\}\}       # Text in double curly braces
-        \s*?\.?$               # Optional whitespace and period at the end
-      }x
-
     def initialize(schema_manifest_file, output_path, urn:,
                    language_code: "eng")
       @schema_manifest_file = File.expand_path(schema_manifest_file)
@@ -64,7 +55,9 @@ module Suma
 
       raise Error, "Schema must have an associated file" unless schema.file
 
-      collection = build_managed_concept_collection(schema)
+      classification = TermClassification.for_schema(id: schema.id,
+                                                      path: schema.file)
+      collection = build_managed_concept_collection(schema, classification)
       output_data(collection)
       collection
     end
@@ -86,7 +79,7 @@ module Suma
       end
     end
 
-    def build_managed_concept_collection(schema)
+    def build_managed_concept_collection(schema, classification)
       source_ref = get_source_ref(schema)
       section_ref = get_section_ref(schema)
 
@@ -102,6 +95,7 @@ module Suma
             entity: entity,
             source_ref: source_ref,
             uuid: localized_concept_id,
+            classification: classification,
           )
 
           managed_data = Glossarist::V3::ManagedConceptData.new.tap do |data|
@@ -126,17 +120,22 @@ module Suma
       end
     end
 
-    def build_localized_concept(schema:, entity:, source_ref:, uuid:)
-      schema_domain = get_domain(schema)
+    def build_localized_concept(schema:, entity:, source_ref:, uuid:,
+                                classification:)
+      schema_domain = classification.domain_for(schema.id)
 
       localized_concept_data = Glossarist::V3::ConceptData.new.tap do |data|
         data.terms = get_entity_terms(entity)
-        data.definition = get_entity_definitions(entity, schema)
+        data.definition = get_entity_definitions(entity, schema, classification)
         data.language_code = @language_code
         data.domain = schema_domain
         data.sources = [source_ref] if source_ref
 
-        notes = get_entity_notes(entity, schema_domain, data.definition)
+        notes = NoteProcessor.call(
+          entity.remarks,
+          definitions: data.definition,
+          xref_to_mention: method(:xref_to_mention),
+        )
         data.notes = notes if notes && !notes.empty?
         data.examples = []
       end
@@ -162,6 +161,10 @@ module Suma
       "{{#{urn},#{display}}}"
     end
 
+    def xref_to_mention(full_ref, display)
+      urn_mention(express_entity_urn(full_ref), display)
+    end
+
     def get_section_ref(schema)
       return nil unless @urn
 
@@ -200,16 +203,6 @@ module Suma
       localities
     end
 
-    def get_domain(schema)
-      prefix = if schema.id.end_with?("_arm", "_mim")
-                 "application module"
-               else
-                 "resource"
-               end
-
-      "#{prefix}: #{schema.id}"
-    end
-
     def get_entity_terms(entity)
       [
         Glossarist::Designation::Base.new(
@@ -220,222 +213,22 @@ module Suma
       ]
     end
 
-    def get_entity_definitions(entity, schema)
-      schema_type = extract_file_type(schema.file)
-      get_domain(schema)
-
-      definition = generate_entity_definition(entity, schema, schema_type)
+    def get_entity_definitions(entity, schema, classification)
+      definition = generate_entity_definition(entity, schema, classification)
       [Glossarist::V3::DetailedDefinition.new(content: definition)]
     end
 
-    def get_entity_notes(entity, schema_domain, definitions)
-      notes = []
-
-      if entity.remarks && !entity.remarks.empty?
-        trimmed_def = trim_definition(entity.remarks)
-        if trimmed_def && !trimmed_def.empty?
-          notes << Glossarist::V3::DetailedDefinition.new(
-            content: convert_express_xref(trimmed_def, schema_domain),
-          )
-        end
-      end
-
-      notes = only_keep_first_sentence(notes)
-      notes = remove_see_content(notes)
-      notes = remove_redundant_note(notes)
-      notes = remove_invalid_references(notes)
-      compare_with_definitions(notes, definitions)
-    end
-
-    def only_keep_first_sentence(notes)
-      notes.each do |note|
-        if note&.content && should_preserve_complete_structure?(note.content)
-          next
-        end
-
-        if note&.content
-          new_content = note.content
-            .split(".\n").first.strip
-            .split(". ").first.strip
-          note.content = new_content.end_with?(".") ? new_content : "#{new_content}."
-        end
-      end
-    end
-
-    def should_preserve_complete_structure?(content)
-      return false if content.nil? || content.empty?
-
-      lines = content.split("\n")
-      first_paragraph = lines.first&.strip
-
-      if first_paragraph&.end_with?(":") && lines.length > 1
-        if first_paragraph.count(".").positive?
-          return false
-        end
-
-        remaining_content = lines[1..].join("\n")
-        return starts_with_list?(remaining_content.strip)
-      end
-
-      false
-    end
-
-    def compare_with_definitions(notes, definitions)
-      if notes&.first&.content == definitions&.first&.content
-        return []
-      end
-
-      notes
-    end
-
-    def remove_invalid_references(notes)
-      notes.reject do |note|
-        note.content.include?("image::") ||
-          note.content.match?(/<<(.*?){1,999}>>/)
-      end
-    end
-
-    def remove_redundant_note(notes)
-      notes.reject do |note|
-        note.content.match?(REDUNDANT_NOTE_REGEX) &&
-          !note.content.include?("\n")
-      end
-    end
-
-    def remove_see_content(notes)
-      notes.each do |note|
-        note.content = note.content.gsub(/\s+\(see(.*?){1,999}\)/, "")
-      end
-    end
-
-    def extract_file_type(filename)
-      match = filename.match(/(arm|mim|bom)_annotated\.exp$/)
-      return "resource" unless match
-
-      {
-        "arm" => "module_arm",
-        "mim" => "module_mim",
-        "bom" => "business_object_model",
-      }[match.captures[0]] || "resource"
-    end
-
-    def starts_with_list?(content)
-      return false if content.nil? || content.empty?
-
-      content.match?(/^\s*[*\-+]\s+/) || content.match?(/^\s*\d+\.\s+/)
-    end
-
-    def trim_definition(definition)
-      return nil if definition.nil? || definition.empty?
-
-      definition_str = definition.is_a?(Array) ? definition.join("\n\n") : definition.to_s
-
-      return nil if definition_str.empty?
-
-      paragraphs = definition_str.split("\n\n")
-      first_paragraph = paragraphs.first
-
-      combined = if paragraphs.length == 1
-                   apply_first_sentence_logic(first_paragraph)
-                 elsif first_paragraph.end_with?(":") && paragraphs.length > 1 && starts_with_list?(paragraphs[1])
-                   complete_list = extract_complete_list(paragraphs, 1)
-                   "#{first_paragraph}\n\n#{complete_list}"
-                 else
-                   apply_first_sentence_logic(first_paragraph)
-                 end
-
-      combined = "#{combined}\n"
-      combined.gsub!(/\n\/\/.*?\n/, "\n")
-      combined.strip!
-
-      express_reference_to_mention(combined)
-    end
-
-    def apply_first_sentence_logic(paragraph)
-      new_content = paragraph
-        .split(".\n").first.strip
-        .split(". ").first.strip
-
-      new_content.end_with?(".") ? new_content : "#{new_content}."
-    end
-
-    def extract_complete_list(paragraphs, start_index)
-      return paragraphs[start_index] if start_index >= paragraphs.length
-
-      combined = paragraphs[start_index].dup
-      current_index = start_index + 1
-      in_continuation_block = combined.include?("--") && !combined.match?(/--.*--/m)
-
-      while current_index < paragraphs.length
-        next_para = paragraphs[current_index]
-
-        if next_para.match?(/^--\s*$/) || next_para.end_with?("--")
-          in_continuation_block = !in_continuation_block
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          next
-        end
-
-        if in_continuation_block
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          next
-        end
-
-        if starts_with_list?(next_para) || is_list_continuation?(next_para)
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          in_continuation_block = true if next_para.include?("--") && !next_para.match?(/--.*--/m)
-        else
-          break
-        end
-      end
-
-      combined
-    end
-
-    def is_list_continuation?(content)
-      return false if content.nil? || content.empty?
-
-      content.match?(/^\+\s*$/) ||
-        content.match?(/^--\s*$/) ||
-        content.match?(/^\s{2,}/) ||
-        content.start_with?("which", "where", "that")
-    end
-
-    def express_reference_to_mention(description)
-      description
-        .gsub(/<<express:([\w.]+)>>/) do |_match|
-          full_ref = Regexp.last_match[1]
-          entity_id = full_ref.split(".").last
-          urn_mention(express_entity_urn(full_ref), entity_id)
-        end.gsub(/<<express:([\w.]+),([\w. ][\w. ]*)>>/) do |_match|
-          full_ref = Regexp.last_match[1]
-          display = Regexp.last_match(2)
-          urn_mention(express_entity_urn(full_ref), display)
-        end
-    end
-
     def entity_name_to_text(entity_id)
       entity_id.downcase.gsub("_", " ")
     end
 
-    def generate_entity_definition(entity, schema, schema_type)
+    def generate_entity_definition(entity, schema, classification)
       return "" if entity.nil?
 
-      entity_type = case schema_type
-                    when "module_arm"
-                      urn_mention(term_urn("general.application_object"),
-                                  "application object")
-                    when "module_mim"
-                      urn_mention(term_urn("express-language.entity_data_type"),
-                                  "entity data type")
-                    when "resource", "business_object_model"
-                      urn_mention(term_urn("express-language.entity_data_type"),
-                                  "entity data type")
-                    else
-                      raise Error, "[suma] encountered unsupported schema_type"
-                    end
+      entity_type = urn_mention(
+        term_urn(classification.entity_term),
+        classification.entity_display,
+      )
 
       entity_ref = urn_mention(term_urn("express-language.entity"), "entity")
 
@@ -454,13 +247,5 @@ module Suma
           "#{entity_name_to_text(entity.id)} #{entity_ref}"
       end
     end
-
-    def convert_express_xref(content, _schema_domain)
-      content.gsub(/<<express:([\w.]+),([\w. ][\w. ]*)>>/) do
-        full_ref = Regexp.last_match(1)
-        display = Regexp.last_match(2)
-        urn_mention(express_entity_urn(full_ref), display)
-      end
-    end
   end
 end

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/suma.rb

@@ -2,6 +2,7 @@
 
 require "expressir"
 require "lutaml/model"
+require "fileutils"
 
 module Suma
   autoload :VERSION, "suma/version"
@@ -10,9 +11,12 @@ module Suma
   autoload :CollectionConfig,       "suma/collection_config"
   autoload :CollectionManifest,     "suma/collection_manifest"
   autoload :EengineConverter,       "suma/eengine_converter"
+  autoload :ExpressReformatter,     "suma/express_reformatter"
   autoload :ExpressSchema,          "suma/express_schema"
+  autoload :LinkValidation,         "suma/link_validation"
   autoload :LinkValidator,          "suma/link_validator"
   autoload :ManifestTraverser,      "suma/manifest_traverser"
+  autoload :NoteProcessor,          "suma/note_processor"
   autoload :RegisterManifestGenerator, "suma/register_manifest_generator"
   autoload :SchemaCategory,         "suma/schema_category"
   autoload :SchemaCollection,       "suma/schema_collection"
@@ -26,6 +30,7 @@ module Suma
   autoload :SchemaTemplate,         "suma/schema_template"
   autoload :SiteConfig,             "suma/site_config"
   autoload :SvgQuality,             "suma/svg_quality"
+  autoload :TermClassification,     "suma/term_classification"
   autoload :TermExtractor,          "suma/term_extractor"
   autoload :ThorExt,                "suma/thor_ext"
   autoload :Urn,                    "suma/urn"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-03 00:00:00.000000000 Z
+date: 2026-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -206,13 +206,17 @@ files:
 - lib/suma/eengine/errors.rb
 - lib/suma/eengine/wrapper.rb
 - lib/suma/eengine_converter.rb
+- lib/suma/express_reformatter.rb
 - lib/suma/express_schema.rb
 - lib/suma/jsdai.rb
 - lib/suma/jsdai/figure.rb
 - lib/suma/jsdai/figure_image.rb
 - lib/suma/jsdai/figure_xml.rb
+- lib/suma/link_validation.rb
 - lib/suma/link_validator.rb
+- lib/suma/link_validator/step.rb
 - lib/suma/manifest_traverser.rb
+- lib/suma/note_processor.rb
 - lib/suma/processor.rb
 - lib/suma/register_manifest_generator.rb
 - lib/suma/schema_category.rb
@@ -235,6 +239,8 @@ files:
 - lib/suma/svg_quality/formatters/terminal_formatter.rb
 - lib/suma/svg_quality/formatters/yaml_formatter.rb
 - lib/suma/svg_quality/report.rb
+- lib/suma/svg_quality/scanner.rb
+- lib/suma/term_classification.rb
 - lib/suma/term_extractor.rb
 - lib/suma/thor_ext.rb
 - lib/suma/urn.rb