suma 0.1.14 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d52aaad6ea81bb43e193cdf0c8e190dfe02f71791e218314a401ae3bea254cd3
-  data.tar.gz: a573936a7eabfad21a4768e6c32869ecc286a2980feeaf8350720b674c604f34
+  metadata.gz: 955161f5bfcd18f46e1fcafafdc05bea5a7310eaa8899b03552e7d22f6f41795
+  data.tar.gz: ff06abeaf766ce76d814079ef718c6deb45e1dd0cae5d89e9d03d203e3a5635d
 SHA512:
-  metadata.gz: d7278d60481858a6507bf97f738df21f0168c60c6453f9de4479545c589450f63c148923be20b31a8bb88e50344adbf8521cfe12a826dbff3103e492782727f3
-  data.tar.gz: b37f9d1f0676607011f6f2059f0b639a563273930eb7961ee52866c04b30df14a7b88076752f4bf2ff4cfd4911a402b547b1c486c42cd9cc381d4e631bc6aac4
+  metadata.gz: 2568587bc47fe2eb892fc7e9f8f4981aceaa06eef4b27ad54cb427e9082cc23571a2c26f8ae9380dfcd52fbb40f38a77ad0d2282e7ada54d8744e3e21edcd301
+  data.tar.gz: 4b8505d33b3eef150aeb76f23548f5983740d1940488a0ee1c7d6e515cd249e5fa74fbdadb72543a91104ca9ce609eb0f374dd01c033db8eee2bb7585525b09f

data/.gitignore

@@ -9,3 +9,7 @@
 
 # rspec failure tracking
 .rspec_status
+.rubocop_todo.yml
+.rubocop-https---*-yml
+.ruby-version
+Gemfile.lock

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2025-04-03 06:48:36 UTC using RuboCop version 1.75.1.
+# on 2025-07-05 22:42:28 UTC using RuboCop version 1.77.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -13,43 +13,32 @@ Gemspec/DuplicatedAssignment:
   Exclude:
     - 'suma.gemspec'
 
-# Offense count: 2
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
-# SupportedHashRocketStyles: key, separator, table
-# SupportedColonStyles: key, separator, table
-# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
-Layout/HashAlignment:
-  Exclude:
-    - 'lib/suma/cli/validate.rb'
-
-# Offense count: 55
+# Offense count: 57
 # This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
+# Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
 Layout/LineLength:
   Exclude:
     - 'lib/suma/cli.rb'
     - 'lib/suma/cli/build.rb'
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
-    - 'lib/suma/collection_manifest.rb'
     - 'lib/suma/processor.rb'
     - 'lib/suma/schema_attachment.rb'
     - 'lib/suma/schema_collection.rb'
     - 'lib/suma/schema_document.rb'
     - 'lib/suma/thor_ext.rb'
+    - 'spec/suma/cli/extract_terms_spec.rb'
     - 'spec/suma/cli/validate_ascii_spec.rb'
     - 'suma.gemspec'
 
-# Offense count: 3
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AllowInHeredoc.
-Layout/TrailingWhitespace:
+# Offense count: 1
+# Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
+Lint/DuplicateBranch:
   Exclude:
-    - 'lib/suma/cli.rb'
-    - 'lib/suma/cli/validate.rb'
+    - 'lib/suma/cli/extract_terms.rb'
 
 # Offense count: 2
 Lint/DuplicateMethods:
@@ -57,10 +46,11 @@ Lint/DuplicateMethods:
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/express_schema.rb'
 
-# Offense count: 15
+# Offense count: 20
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Exclude:
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/schema_attachment.rb'
@@ -73,15 +63,16 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 64
 
-# Offense count: 4
+# Offense count: 8
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/CyclomaticComplexity:
   Exclude:
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/thor_ext.rb'
 
-# Offense count: 21
+# Offense count: 30
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 107
@@ -91,10 +82,11 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Max: 6
 
-# Offense count: 2
+# Offense count: 5
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/PerceivedComplexity:
   Exclude:
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
 
@@ -113,12 +105,12 @@ Performance/CollectionLiteralInLoop:
   Exclude:
     - 'spec/suma/cli_spec.rb'
 
-# Offense count: 3
+# Offense count: 9
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
-  Max: 16
+  Max: 44
 
-# Offense count: 5
+# Offense count: 6
 RSpec/MultipleExpectations:
   Max: 12
 
@@ -136,3 +128,9 @@ Style/EmptyElse:
 # SupportedStyles: annotated, template, unannotated
 Style/FormatStringToken:
   EnforcedStyle: unannotated
+
+# Offense count: 1
+# Configuration parameters: Max.
+Style/SafeNavigationChainLength:
+  Exclude:
+    - 'lib/suma/cli/extract_terms.rb'

data/README.adoc

@@ -331,8 +331,10 @@ out which schemas the document includes.
 
 === Extract terms command
 
-The `suma extract_terms` command extracts terms from SCHEMA_MANIFEST_FILE and
-generates Glossarist v2 dataset in the output directory.
+The `suma extract_terms` command extracts terms from EXPRESS schemas and
+generates a Glossarist v2 dataset in the output directory. This command processes
+various types of STEP schemas and creates standardized terminology datasets
+suitable for glossary and dictionary applications.
 
 [source,sh]
 ----
@@ -341,38 +343,80 @@ $ suma extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH [options]
 
 Parameters:
 
-`SCHEMA_MANIFEST_FILE`:: Path to SCHEMA_MANIFEST_FILE
+`SCHEMA_MANIFEST_FILE`:: Path to the schema manifest file that lists all schemas
+to process (e.g., "schemas-smrl-all.yml")
 
-`GLOSSARIST_OUTPUT_PATH`:: Path to the output directory for the Glossarist v2
-dataset
+`GLOSSARIST_OUTPUT_PATH`:: Path to the output directory where the Glossarist v2
+dataset will be generated
 
 Options:
 
-`language_code`:: Language code for the Glossarist (default: "eng")
+`--language_code`, `-l`:: Language code for the Glossarist dataset (default: "eng")
 
+==== Supported schema types
+
+The command supports extraction from the following EXPRESS schema types:
+
+* **ARM (Application Reference Model)** - Application module schemas ending with `_arm`
+* **MIM (Module Implementation Model)** - Application module schemas ending with `_mim`
+* **Resource schemas** - General resource schemas
+* **BOM (Business Object Model)** - Business object model schemas ending with `_bom`
+
+==== Extracted terms
+
+The command extracts the following types of terms from EXPRESS schemas:
+
+* **Entities** - EXPRESS entity definitions with their attributes and relationships
+* **Types** - EXPRESS type definitions including enumerations and select types
+* **Functions** - EXPRESS function definitions
+* **Procedures** - EXPRESS procedure definitions
+* **Constants** - EXPRESS constant definitions
+
+Each extracted term includes:
+
+* Unique identifier based on schema and term name
+* Term definition and description
+* Source schema information
+* Appropriate domain classification (application module, resource, or business object model)
+
+==== Output format
+
+The command generates a Glossarist v2 compliant dataset with:
+
+* `concept/` directory containing concept definition files
+* `localized_concept/` directory containing localized concept files
+* YAML format following Glossarist v2 schema specifications
+* Proper cross-references and citations to source schemas
+
+.To extract terms from a schema manifest file
 [example]
 ====
-.To extract terms from SCHEMA_MANIFEST_FILE
 [source,sh]
 ----
-$ bundle exec suma extract_terms path/to/schemas-smrl-all.yml glossarist_output
-# => generates glossarist_output/concept/foo.yaml and glossarist_output/localized_concept/bar.yaml
+$ bundle exec suma extract_terms schemas-smrl-all.yml glossarist_output
+# => generates glossarist_output/concept/*.yaml and glossarist_output/localized_concept/*.yaml
 ----
+====
 
-.To extract terms from SCHEMA_MANIFEST_FILE with language code "fra"
+.To extract terms from a specific schema subset
+[example]
+====
 [source,sh]
 ----
-$ bundle exec suma extract_terms path/to/schemas-smrl-all.yml glossarist_output -l fra
-# => generates glossarist_output/concept/foo.yaml and glossarist_output/localized_concept/bar.yaml
+$ bundle exec suma extract_terms schemas-activity-modules.yml terms_output
+# => processes only schemas listed in the manifest file
 ----
 ====
 
+The generated dataset is meant to be used for ISO 10303-2.
+
 
 == Usage: Ruby
 
 === General
 
-Suma can be used programmatically in your Ruby applications. The following examples demonstrate common usage patterns.
+Suma can be used programmatically in your Ruby applications. The following
+examples demonstrate common usage patterns.
 
 === Building collections
 

data/lib/suma/cli/extract_terms.rb

@@ -4,7 +4,6 @@ require "thor"
 require_relative "../thor_ext"
 require "fileutils"
 require "expressir"
-require "yaml"
 require "securerandom"
 require "glossarist"
 
@@ -13,29 +12,30 @@ module Suma
     # ExtractTerms command using Expressir to extract terms into the
     # Glossarist v2 format
     class ExtractTerms < Thor
+      # Matches patterns like "A thing is a type of {{entity}}." or
+      # "An object is a type of a {{entity}}"
+      REDUNDANT_NOTE_REGEX =
+        %r{
+          ^An?                   # Starts with "A" or "An"
+          \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
+          (\sa|\san)?            # Optional " a" or " an"
+          \s\{\{[^\}]*\}\}       # Text in double curly braces
+          \s*?\.?$               # Optional whitespace and period at the end
+        }x
+
       desc "extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
            "Extract terms from SCHEMA_MANIFEST_FILE into " \
            "Glossarist v2 format"
       option :language_code, type: :string, default: "eng", aliases: "-l",
                              desc: "Language code for the Glossarist"
 
-      YAML_FILE_EXTENSIONS = [".yaml", ".yml"].freeze
-
-      def extract_terms(schema_manifest_file, output_path) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      def extract_terms(schema_manifest_file, output_path)
         language_code = options[:language_code]
         schema_manifest_file = File.expand_path(schema_manifest_file)
 
-        if File.file?(schema_manifest_file)
-          unless File.exist?(schema_manifest_file)
-            raise Errno::ENOENT, "Specified SCHEMA_MANIFEST_FILE " \
-                                 "`#{schema_manifest_file}` not found."
-          end
-
-          if !YAML_FILE_EXTENSIONS.include?(File.extname(schema_manifest_file))
-            raise ArgumentError, "Specified SCHEMA_MANIFEST_FILE " \
-                                 "`#{schema_manifest_file}` " \
-                                 "is not a YAML file."
-          end
+        unless File.exist?(schema_manifest_file)
+          raise Errno::ENOENT, "Specified SCHEMA_MANIFEST_FILE " \
+                               "`#{schema_manifest_file}` not found."
         end
 
         run(schema_manifest_file, output_path, language_code)
@@ -44,152 +44,185 @@ module Suma
       private
 
       def run(schema_manifest_file, output_path, language_code = "eng")
-        exp_files = get_exp_files(schema_manifest_file)
-
-        exp_files.map do |exp_file|
+        get_exp_files(schema_manifest_file).map do |exp_file|
           extract(exp_file, output_path, language_code)
         end
       end
 
       def get_exp_files(schema_manifest_file)
-        data = YAML.safe_load(
-          File.read(schema_manifest_file, encoding: "UTF-8"),
-          permitted_classes: [Date, Time, Symbol],
-          permitted_symbols: [],
-          aliases: true,
-        )
-
-        paths = data["schemas"].values.filter_map { |v| v["path"] }
+        config = Expressir::SchemaManifest.from_file(schema_manifest_file)
+        paths = config.schemas.map(&:path)
 
         if paths.empty?
           raise Errno::ENOENT, "No EXPRESS files found in " \
                                "`#{schema_manifest_file}`."
         end
 
-        # resolve paths relative to the directory of the schema manifest file
-        paths.map do |path|
-          File.expand_path(path, File.dirname(schema_manifest_file))
-        end
+        paths
       end
 
-      def extract(exp_file, output_path, language_code) # rubocop:disable Metrics/AbcSize
-        puts "Processing EXPRESS file: #{exp_file}"
+      def extract(exp_file, output_path, language_code)
+        exp_path_rel = Pathname.new(exp_file).relative_path_from(Pathname.getwd)
+        puts "Building terms: #{exp_path_rel}"
+
         repo = Expressir::Express::Parser.from_file(exp_file)
         schema = get_default_schema(repo)
 
+        unless schema.file
+          raise Error.new("Schema must have an associated file")
+        end
+
         collection = build_managed_concept_collection(
           schema, language_code
         )
 
-        output_data(collection, output_path, exp_file)
+        output_data(collection, output_path)
       end
 
-      def output_data(collection, output_path, exp_file)
+      def output_data(collection, output_path)
         unless File.exist?(output_path)
           FileUtils.mkdir_p(File.expand_path(output_path))
         end
 
-        puts "Saving collection to files in: #{File.expand_path(output_path)}"
+        puts "Saving collection to files in: #{output_path}"
         collection.save_to_files(File.expand_path(output_path))
 
-        puts "Processing EXPRESS file: #{exp_file}...Done."
         collection
       end
 
-      def build_managed_concept_collection(schema, language_code) # rubocop:disable Metrics/AbcSize
-        managed_concept_data = Glossarist::ManagedConceptData.new
-        managed_concept_data.id = get_identifier(schema)
-
-        localized_concept_id = SecureRandom.uuid
-        localized_concept = build_localized_concept(
-          schema, language_code, localized_concept_id
-        )
-
-        managed_concept_data
-          .localizations[localized_concept.language_code] = localized_concept
-
-        managed_concept_data.localized_concepts = {
-          localized_concept.language_code => localized_concept_id,
-        }
-
-        managed_concept = Glossarist::ManagedConcept.new
-        managed_concept.uuid = SecureRandom.uuid
-        managed_concept.data = managed_concept_data
-
-        collection = Glossarist::ManagedConceptCollection.new
-        collection.store(managed_concept)
-        collection
+      def build_managed_concept_collection(schema, language_code)
+        Glossarist::ManagedConceptCollection.new.tap do |collection|
+          # Extract schema-level citation data once to reuse across all entities
+          source_ref = get_source_ref(schema)
+
+          # Create one concept per entity
+          schema.entities.each do |entity|
+            localized_concept = build_localized_concept(
+              schema: schema,
+              entity: entity,
+              language_code: language_code,
+              source_ref: source_ref,
+            )
+
+            managed_data = Glossarist::ManagedConceptData.new.tap do |data|
+              data.id = get_entity_identifier(schema, entity)
+
+              # TODO: Why do we need both localizations and localized_concepts??
+              data.localizations[language_code] = localized_concept
+              # uuid is automatically set from the serialization of the object
+              data.localized_concepts = {
+                language_code => get_localized_concept_identifier(
+                  schema, entity, language_code
+                ),
+              }
+            end
+
+            managed_concept = Glossarist::ManagedConcept.new.tap do |concept|
+              # uuid is automatically set from the serialization of the object
+              concept.id = get_entity_identifier(schema, entity)
+              concept.uuid = concept.id
+              concept.data = managed_data
+            end
+
+            collection.store(managed_concept)
+          end
+        end
       end
 
-      def build_localized_concept(schema, language_code, localized_concept_id) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
+      def build_localized_concept(schema:, entity:, language_code:, source_ref:)
         schema_domain = get_domain(schema)
 
-        localized_concept_data = Glossarist::ConceptData.new
-        localized_concept_data.terms = get_terms(schema) || []
-        localized_concept_data.definition = get_definitions(schema) || []
-        localized_concept_data.notes = get_notes(schema, schema_domain) || []
-        localized_concept_data.examples = get_examples(schema,
-                                                       schema_domain) || []
-        localized_concept_data.language_code = language_code
-        localized_concept_data.domain = schema_domain
-        localized_concept_data.sources = get_source_ref(schema) || []
+        localized_concept_data = Glossarist::ConceptData.new.tap do |data|
+          data.terms = get_entity_terms(entity)
+          data.definition = get_entity_definitions(entity, schema)
+          data.language_code = language_code
+          data.domain = schema_domain
+          data.sources = [source_ref] if source_ref
 
-        localized_concept = Glossarist::LocalizedConcept.new
-        localized_concept.data = localized_concept_data
+          # Only assign optional fields if they have content
+          notes = get_entity_notes(entity, schema_domain, data.definition)
+          data.notes = notes if notes && !notes.empty?
+
+          # examples = get_entity_examples(entity, schema_domain)
+          # data.examples = examples if examples && !examples.empty?
+          data.examples = []
+        end
 
-        localized_concept.uuid = localized_concept_id
-        localized_concept
+        Glossarist::LocalizedConcept.new.tap do |concept|
+          concept.data = localized_concept_data
+        end
       end
 
+      # We only deal with 1 schema
       def get_default_schema(repo)
         repo.schemas.first
       end
 
-      def get_identifier(schema)
-        remark_item = schema.remark_items.find do |s|
-          s.id == "__identifier"
-        end
-        remark_item.remarks.first || SecureRandom.uuid
+      def find_remark_value(schema, remark_id)
+        schema.remark_items.find { |s| s.id == remark_id }&.remarks&.first
       end
 
-      def get_title(schema)
-        remark_item = schema.remark_items.find do |s|
-          s.id == "__title"
-        end
-        remark_item.remarks.first
+      def get_entity_identifier(schema, entity)
+        "#{schema.id}.#{entity.id}"
+      end
+
+      def get_localized_concept_identifier(schema, entity, lang)
+        "#{schema.id}.#{entity.id}-#{lang}"
       end
 
       def get_source_ref(schema)
-        remark_item = schema.remark_items.find do |s|
-          s.id == "__published_in"
+        origin = Glossarist::Citation.new.tap do |citation|
+          citation.ref = "ISO 10303"
+          custom_locality = build_custom_locality(schema)
+
+          unless custom_locality.empty?
+            citation.custom_locality = custom_locality
+          end
         end
-        ref = remark_item&.remarks&.first
 
-        if ref
-          Glossarist::ConceptSource.new(
-            type: "authoritative",
-            origin: Glossarist::Citation.new(ref: ref),
+        Glossarist::ConceptSource.new(type: "authoritative", origin: origin)
+      end
+
+      # SCHEMA action_schema
+      # '{iso standard 10303 part(41) version(9) object(1) action-schema(1)}';
+      def build_custom_locality(schema)
+        [].tap do |localities|
+          # Add schema name
+          localities << Glossarist::CustomLocality.new(
+            name: "schema",
+            value: schema.id,
           )
+
+          # Add version if available
+          version_item = schema.version.items.detect { |i| i.name == "version" }
+          if version_item
+            localities << Glossarist::CustomLocality.new(
+              name: "version",
+              value: version_item.value,
+            )
+          end
         end
       end
 
+      # TODO: What if this was a "bom"?
       def get_domain(schema)
-        prefix = module?(schema) ? "application module" : "resource"
-        "#{prefix}: #{schema.id}"
-      end
-
-      def module?(schema)
-        remark_item = schema.remark_items.find do |s|
-          s.id == "__schema_file"
-        end
+        prefix = if mim?(schema.id) || arm?(schema.id)
+                   "application module"
+                 else
+                   "resource"
+                 end
 
-        File.basename(remark_item.remarks.first, ".*") == "module"
+        "#{prefix}: #{schema.id}"
       end
 
       def arm?(schema_id)
         schema_id.end_with?("_arm")
       end
 
+      def mim?(schema_id)
+        schema_id.end_with?("_mim")
+      end
+
       def get_terms(schema)
         schema_title = get_title(schema)
         if schema_title
@@ -203,69 +236,291 @@ module Suma
         end
       end
 
-      def get_schema_type(schema)
-        return "resource" if !module?(schema)
+      def get_entity_terms(entity)
+        # For now, use the entity ID as the term
+        # This could be enhanced to look for entity-specific title remark items
+        [
+          Glossarist::Designation::Base.new(
+            designation: entity.id,
+            type: "expression",
+            normative_status: "preferred",
+          ),
+        ]
+      end
 
-        return "arm" if arm?(schema.id)
+      def get_entity_definitions(entity, schema)
+        schema_type = extract_file_type(schema.file)
+        schema_domain = get_domain(schema)
 
-        "min"
+        definition = generate_entity_definition(entity, schema_domain,
+                                                schema_type)
+        [Glossarist::DetailedDefinition.new(content: definition)]
       end
 
-      def get_definitions(schema)
-        type = get_schema_type(schema)
-        subtype = get_subtype_of(schema)
+      def get_entity_notes(entity, schema_domain, definitions)
+        puts "Extracting notes for entity: #{entity.id}"
+        notes = []
+
+        notes = add_entity_notes(entity, schema_domain, notes)
+        # notes = add_other_notes(entity, schema_domain, notes)
+        notes = only_keep_first_sentence(notes)
+        notes = remove_see_content(notes)
+        notes = remove_redundant_note(notes)
+        notes = remove_invalid_references(notes)
+        compare_with_definitions(notes, definitions)
+      end
 
-        represent_str = "that represents the #{get_title(schema)} {{entity}}"
-        if subtype
-          represent_str = "that is a type of #{subtype} #{represent_str}"
+      def add_entity_notes(entity, schema_domain, notes)
+        # Add trimmed definition from entity description as first note
+        if entity.remarks && !entity.remarks.empty?
+          trimmed_def = trim_definition(entity.remarks)
+          if trimmed_def && !trimmed_def.empty?
+            notes << Glossarist::DetailedDefinition.new(
+              content: convert_express_xref(trimmed_def, schema_domain),
+            )
+          end
         end
 
-        definition = case type
-                     when "arm"
-                       "{{application object}} #{represent_str}"
-                     else
-                       "{{entity data type}} #{represent_str}"
-                     end
-        [Glossarist::DetailedDefinition.new(content: definition)]
-      end
-
-      def get_subtype_of(schema)
-        schema.entities.first&.subtype_of&.first&.id # rubocop:disable Style/SafeNavigationChainLength
+        notes.compact
       end
 
-      # get entities remarks and remark items with id `__note` as notes
-      def get_notes(schema, schema_domain) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
-        notes = schema.entities&.map do |entity|
-          [
-            entity.remarks,
-            entity.remark_items&.select do |ri|
-              ri.id == "__note"
-            end&.map(&:remarks),
-          ]
-        end&.flatten&.compact
+      def add_other_notes(entity, schema_domain, notes)
+        # Add other notes from entity remarks
+        other_notes = [
+          entity.remark_items&.select do |ri|
+            ri.id == "__note"
+          end&.map(&:remarks),
+        ].flatten.compact
 
-        notes&.map do |note|
-          Glossarist::DetailedDefinition.new(
+        other_notes.each do |note|
+          notes << Glossarist::DetailedDefinition.new(
             content: convert_express_xref(note, schema_domain),
           )
         end
+
+        notes
       end
 
-      # get entities remark items with id `__example` as examples
-      def get_examples(schema, schema_domain) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
-        examples = schema.entities&.map do |entity|
-          entity.remark_items&.select do |ri|
-            ri.id == "__example"
-          end&.map(&:remarks)
-        end&.flatten&.compact
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 1. First sentence in first paragraph of the entity description
+      # (in EXPRESS remark) becomes NOTE 1 in ISO 10303-2 of the entity.
+      def only_keep_first_sentence(notes)
+        notes.each do |note|
+          # Split by period and take the first sentence
+          # Avoid splitting by pattern like "abc.def"
+          if note&.content
+            new_content = note.content
+              .split(".\n").first.strip
+              .split(". ").first.strip
+            note.content = if new_content.end_with?(".")
+                             new_content
+                           else
+                             "#{new_content}."
+                           end
+          end
+        end
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 2. If this first sentence matches the 7-word magic sentence
+      # (2-3 forms of that), it is discarded so there will not be a NOTE 1.
+      def compare_with_definitions(notes, definitions)
+        if notes&.first&.content == definitions&.first&.content
+          # Discarding first note as it matches the definition
+          return []
+        end
+
+        notes
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 3. No reference to any types or attribute or figures allowed in first
+      # sentence. Entity references “{{…}}” are allowed.
+      def remove_invalid_references(notes)
+        notes.reject do |note|
+          note.content.include?("image::") ||
+            note.content.match?(/<<(.*?){1,999}>>/)
+        end
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 4. Entity notes and examples in EXPRESS remarks are NOT represented in
+      # part 2.
+      def remove_redundant_note(notes)
+        notes.reject do |note|
+          note.content.match?(REDUNDANT_NOTE_REGEX) &&
+            !note.content.include?("\n")
+        end
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 5. If the sentence contains “\s+(see …)”, the contents including the
+      # parentheses are removed.
+      def remove_see_content(notes)
+        notes.each do |note|
+          note.content = note.content.gsub(/\s+\(see(.*?){1,999}\)/, "")
+        end
+      end
+
+      def get_entity_examples(entity, schema_domain)
+        examples = entity.remark_items&.select do |ri|
+          ri.id == "__example"
+        end&.map(&:remarks)&.flatten&.compact || []
 
-        examples&.map do |example|
+        examples.map do |example|
           Glossarist::DetailedDefinition.new(
             content: convert_express_xref(example, schema_domain),
           )
         end
       end
 
+      def extract_file_type(filename)
+        match = filename.match(/(arm|mim|bom)_annotated\.exp$/)
+        return "resource" unless match
+
+        {
+          "arm" => "module_arm",
+          "mim" => "module_mim",
+          "bom" => "business_object_model",
+        }[match.captures[0]] || "resource"
+      end
+
+      def get_schema_type(schema)
+        return "mim" if mim?(schema.id)
+        return "arm" if arm?(schema.id)
+        return "bom" if bom?(schema.id)
+
+        "resource"
+      end
+
+      def bom?(schema_id)
+        schema_id.end_with?("_bom")
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def combine_paragraphs(full_paragraph, next_paragraph)
+        # If full_paragraph already contains a period, extract that.
+        if m = full_paragraph.match(/\A(?<inner_first>[^\n]*?\.)\s/)
+          # puts "CONDITION 1"
+          if m[:inner_first]
+            return m[:inner_first]
+          else
+            return full_paragraph
+          end
+        end
+
+        # If full_paragraph ends with a period, this is the last.
+        if /\.\s*\Z/.match?(full_paragraph)
+          # puts "CONDITION 2"
+          return full_paragraph
+        end
+
+        # If next_paragraph is a list
+        if next_paragraph.start_with?("*")
+          # puts "CONDITION 3"
+          return "#{full_paragraph}\n\n#{next_paragraph}"
+        end
+
+        # If next_paragraph is a continuation of a list
+        if next_paragraph.start_with?("which", "that")
+          # puts "CONDITION 4"
+          return "#{full_paragraph}\n\n#{next_paragraph}"
+        end
+
+        # puts "CONDITION 5"
+        full_paragraph
+      end
+
+      def trim_definition(definition)
+        return nil if definition.nil? || definition.empty?
+
+        # Handle case where definition is an array
+        definition_str = if definition.is_a?(Array)
+                           definition.join("\n\n")
+                         else
+                           definition.to_s
+                         end
+
+        return nil if definition_str.empty?
+
+        # Unless the first paragraph ends with "between" and is followed by a
+        # list, don't split
+        paragraphs = definition_str.split("\n\n")
+
+        # puts paragraphs.inspect
+
+        first_paragraph = paragraphs.first
+
+        combined = if paragraphs.length > 1
+                     paragraphs[1..].inject(first_paragraph) do |acc, p|
+                       combine_paragraphs(acc, p)
+                     end
+                   else
+                     combine_paragraphs(first_paragraph, "")
+                   end
+
+        # puts "combined--------- #{combined}"
+
+        # Remove comments until end of line
+        combined = "#{combined}\n"
+        combined.gsub!(/\n\/\/.*?\n/, "\n")
+        combined.strip!
+
+        express_reference_to_mention(combined)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # Replace `<<express:{schema}.{entity}>>` with {{entity}}
+      # and `<<express:{schema}.{entity},{render}>>` with {{entity,render}}
+      def express_reference_to_mention(description)
+        # TODO: Use Expressir to check whether the "entity" is really an
+        # EXPRESS ENTITY. If not, skip the mention.
+        description
+          .gsub(/<<express:([^,]+)>>/) do |_match|
+            "{{#{Regexp.last_match[1].split('.').last}}}"
+          end.gsub(/<<express:([^,]+),([^>]+)>>/) do |_match|
+            "{{#{Regexp.last_match[1].split('.').last}," \
+              "#{Regexp.last_match[2]}}}"
+          end
+      end
+
+      def entity_name_to_text(entity_id)
+        entity_id.downcase.gsub("_", " ")
+      end
+
+      # rubocop:disable Layout/LineLength
+      def generate_entity_definition(entity, _domain, schema_type)
+        return "" if entity.nil?
+
+        # See: metanorma/iso-10303-2#90
+        entity_type = case schema_type
+                      when "module_arm"
+                        "{{application object}}"
+                      when "module_mim"
+                        "{{entity data type}}"
+                      when "resource", "business_object_model"
+                        "{{entity data type}}"
+                      else
+                        raise Error.new("[suma] encountered unsupported schema_type")
+                      end
+
+        if entity.subtype_of.empty?
+          "#{entity_type} " \
+            "that represents the " \
+            "#{entity_name_to_text(entity.id)} {{entity}}"
+        else
+          entity_subtypes = entity.subtype_of.map do |e|
+            "{{#{e.id}}}"
+          end
+
+          "#{entity_type} that is a type of " \
+            "#{entity_subtypes.join(' and ')} " \
+            "that represents the " \
+            "#{entity_name_to_text(entity.id)} {{entity}}"
+        end
+      end
+      # rubocop:enable Layout/LineLength
+
       def convert_express_xref(content, schema_domain)
         content.gsub(/<<express:(.*),(.*)>>/) do
           "{{<#{schema_domain}>" \
@@ -273,13 +528,8 @@ module Suma
         end
       end
 
-      def get_concept_filename(concept)
-        identifier = concept["data"]["identifier"]
-        "#{sanitize_string(identifier)}.yaml"
-      end
-
-      def sanitize_string(str)
-        str.gsub(" ", "_").gsub("/", "_").gsub(":", "_")
+      def id_from_designation(designation)
+        designation.gsub(" ", "_").gsub("/", "_").gsub(":", "_")
       end
     end
   end

data/lib/suma/cli/reformat.rb

@@ -57,7 +57,7 @@ module Suma
         # containing '(*text*)' inside
         comments = file_content.scan(/\(\*"(.*?)\n\*\)/m).map(&:first)
 
-        if comments.count.positive?
+        if comments.any?
           content_without_comments = file_content.gsub(/\(\*".*?\n\*\)/m, "")
 
           # remove extra newlines

data/lib/suma/cli/validate_links.rb

@@ -2,6 +2,7 @@
 
 require "thor"
 require_relative "../utils"
+require "expressir"
 
 module Suma
   module Cli
@@ -43,7 +44,6 @@ module Suma
         # Lazy-load dependencies only when this command is actually used
         require "expressir"
         require "ruby-progressbar"
-        require_relative "../schema_config"
         require "pathname"
       end
 
@@ -74,7 +74,7 @@ module Suma
 
       # Load and initialize the schemas configuration
       def load_schemas_config(schemas_file_path)
-        schemas_config = Suma::SchemaConfig::Config.from_yaml(File.read(schemas_file_path))
+        schemas_config = Expressir::SchemaManifest.from_yaml(File.read(schemas_file_path))
         # Ensure the config is initialized with the correct path to resolve relative paths
         schemas_config.set_initial_path(schemas_file_path.to_s)
         schemas_config

data/lib/suma/collection_manifest.rb

@@ -3,6 +3,7 @@
 require "metanorma/cli"
 require "metanorma/cli/collection"
 require "metanorma/collection/collection"
+require "expressir"
 
 module Suma
   class CollectionManifest < Metanorma::Collection::Config::Manifest
@@ -36,7 +37,7 @@ module Suma
     end
 
     def export_schema_config(path)
-      export_config = @schema_config || Suma::SchemaConfig::Config.new
+      export_config = @schema_config || Expressir::SchemaManifest.new
       return export_config unless entry
 
       entry.each do |x|
@@ -127,7 +128,7 @@ module Suma
       if File.basename(file) == "collection.yml"
         schemas_yaml_path = File.join(File.dirname(file), "schemas.yaml")
         if schemas_yaml_path && File.exist?(schemas_yaml_path)
-          @schema_config = Suma::SchemaConfig::Config.from_file(schemas_yaml_path)
+          @schema_config = Expressir::SchemaManifest.from_file(schemas_yaml_path)
         end
       end
     end

data/lib/suma/processor.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "schema_config"
 require_relative "schema_collection"
 require_relative "utils"
 require_relative "collection_config"

data/lib/suma/schema_attachment.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require "expressir"
 # require "metanorma/cli"
-require_relative "schema_config"
 
 module Suma
   class SchemaAttachment
@@ -68,8 +68,8 @@ module Suma
 
     def to_config(path: nil)
       # return @config unless @config
-      @config = SchemaConfig::Config.new
-      @config.schemas << SchemaConfig::Schema.new(
+      @config = Expressir::SchemaManifest.new
+      @config.schemas << Expressir::SchemaManifestEntry.new(
         id: @schema.id,
         path: @schema.path,
       )

data/lib/suma/schema_collection.rb

@@ -3,7 +3,7 @@
 require_relative "express_schema"
 require_relative "schema_attachment"
 require_relative "schema_document"
-require_relative "schema_config"
+require "expressir"
 require_relative "utils"
 
 module Suma
@@ -19,7 +19,7 @@ module Suma
       @output_path_docs = Pathname.new(output_path_docs || Dir.pwd).expand_path
       @output_path_schemas = Pathname.new(output_path_schemas || Dir.pwd).expand_path
       @config = config
-      @config ||= config_yaml && SchemaConfig::Config.from_file(config_yaml)
+      @config ||= config_yaml && Expressir::SchemaManifest.from_file(config_yaml)
       @manifest = manifest
     end
 

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.1.14"
+  VERSION = "0.1.16"
 end

data/lib/suma.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "expressir"
 require_relative "suma/version"
 require_relative "suma/processor"
 

data/suma.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.require_paths = ["lib"]
 
   spec.add_dependency "expressir", "~> 2.1"
+  spec.add_dependency "glossarist", "~> 2.3.7"
   spec.add_dependency "lutaml-model", "~> 0.7"
   spec.add_dependency "metanorma-cli"
   spec.add_dependency "plurimath"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.16
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-06-20 00:00:00.000000000 Z
+date: 2025-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: glossarist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.7
 - !ruby/object:Gem::Dependency
   name: lutaml-model
   requirement: !ruby/object:Gem::Requirement
@@ -146,9 +160,6 @@ files:
 - lib/suma/processor.rb
 - lib/suma/schema_attachment.rb
 - lib/suma/schema_collection.rb
-- lib/suma/schema_config.rb
-- lib/suma/schema_config/config.rb
-- lib/suma/schema_config/schema.rb
 - lib/suma/schema_document.rb
 - lib/suma/site_config.rb
 - lib/suma/thor_ext.rb

data/lib/suma/schema_config/config.rb

@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-require "lutaml/model"
-require_relative "schema"
-require_relative "../utils"
-
-module Suma
-  module SchemaConfig
-    class Config < Lutaml::Model::Serializable
-      attribute :schemas, Schema, collection: true, initialize_empty: true
-      attribute :path, Lutaml::Model::Type::String
-      attr_accessor :output_path
-
-      def initialize(**args)
-        @path = path_relative_to_absolute(path) if path
-        super
-      end
-
-      def base_path
-        File.dirname(@path)
-      end
-
-      yaml do
-        map "schemas", with: { from: :schemas_from_yaml, to: :schemas_to_yaml }
-      end
-
-      def self.from_file(path)
-        from_yaml(File.read(path)).tap do |x|
-          x.set_initial_path(path)
-        end
-      end
-
-      def to_file(to_path = path)
-        File.write(to_path, to_yaml)
-      end
-
-      def set_initial_path(new_path)
-        @path = path_relative_to_absolute(new_path)
-        schemas.each do |schema|
-          schema.path = path_relative_to_absolute(schema.path)
-          schema.container_path = File.expand_path(@path)
-        end
-      end
-
-      def schemas_from_yaml(model, value)
-        model.schemas = value.map do |k, v|
-          Schema.new(id: k, path: path_relative_to_absolute(v["path"]))
-        end
-      end
-
-      def schemas_to_yaml(model, doc)
-        doc["schemas"] = model.schemas.sort_by(&:id).to_h do |schema|
-          # We are outputting the schemas collection file to the directory where
-          # the collection config is at (assumed to be Dir.pwd), not to the
-          # directory we sourced the manifest from, e.g.
-          # documents/iso-10303-41/schemas.yaml.
-
-          # So the schema.container_path = @config.path is not
-          # in fact needed, as the files are already absolute. This notion of
-          # using @path to create relative paths was misconceived
-          [
-            schema.id,
-            {
-              "path" => path_absolute_to_relative(
-                schema.path,
-                model.output_path || Dir.pwd,
-              ),
-            },
-          ]
-        end
-      end
-
-      def path_relative_to_absolute(relative_path)
-        eval_path = Pathname.new(relative_path)
-        return relative_path if eval_path.absolute?
-
-        # Or based on current working directory?
-        return relative_path unless @path
-
-        # ... but if this calculates path, we end up expanding it anyway
-
-        Pathname.new(File.dirname(@path)).join(eval_path).expand_path.to_s
-      end
-
-      def path_absolute_to_relative(absolute_path, container_path)
-        container_path ||= @path
-        return absolute_path unless container_path
-
-        p = Pathname.new(container_path)
-        container = p.directory? ? p.to_s : p.dirname
-        Pathname.new(absolute_path).relative_path_from(container).to_s
-      end
-
-      def update_path(new_path)
-        if @path.nil?
-          @path = new_path
-          return @path
-        end
-
-        old_base_path = File.dirname(@path)
-        new_base_path = File.dirname(new_path)
-        update_schema_path(old_base_path, new_base_path)
-
-        @path = new_path
-      end
-
-      def concat(another_config)
-        unless another_config.is_a?(self.class)
-          raise StandardError, "Can only concat a SchemaConfig::Config object."
-        end
-
-        # We need to update the relative paths when paths exist
-        if path && another_config.path && path != another_config.path
-          new_config = another_config.dup
-          new_config.update_path(path)
-        end
-
-        schemas.concat(another_config.schemas)
-      end
-
-      def save_to_path(filename)
-        new_config = dup.tap do |c|
-          c.path = filename
-          c.update_path(File.dirname(filename))
-          c.output_path = filename
-        end
-
-        File.open(filename, "w") do |f|
-          Utils.log "Writing #{filename}..."
-          f.write(new_config.to_yaml)
-          Utils.log "Done."
-        end
-      end
-
-      def update_schema_path(old_base_path, new_base_path)
-        schemas.each do |schema|
-          schema_path = Pathname.new(schema.path)
-          next if schema_path.absolute?
-
-          schema_path = (Pathname.new(old_base_path) + schema_path).cleanpath
-          # This is the new relative schema_path
-          schema.path = schema_path.relative_path_from(new_base_path)
-        end
-      end
-    end
-  end
-end

data/lib/suma/schema_config/schema.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require "lutaml/model"
-
-module Suma
-  module SchemaConfig
-    class Schema < Lutaml::Model::Serializable
-      attribute :id, Lutaml::Model::Type::String
-      attribute :path, Lutaml::Model::Type::String
-      # attribute :schemas_only, Lutaml::Model::Type::Boolean
-
-      # container_path is a copy of Suma::SchemaConfig::Config.path,
-      # used to resolve the path of each schema within
-      # Suma::SchemaConfig::Config.schemas,
-      # when Suma::SchemaConfig::Config.schemas is recursively flattened
-      attr_accessor :container_path
-    end
-  end
-end

data/lib/suma/schema_config.rb

@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "schema_config/schema"
-require_relative "schema_config/config"