suma 0.1.13 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fcc000e30cb69dd5ab1d326cbcf06ca4fe2c2855a7368f2a0387a369c15721c
-  data.tar.gz: 3873c8355f19b4ccb2f8b8d3530eeac2ad81e65e880e44b97765413377ff71a1
+  metadata.gz: 52836adb9e003f9d760a6c015207379be88b8821855d64388a5f30d09474de7e
+  data.tar.gz: d7427de3e6dc21291671122bd07bc23c739e4f59f031e3b02988f89254767d00
 SHA512:
-  metadata.gz: 0a18e25ee8e47d39028d0bd9bd5dd9c2f602bf540f77b3729f402bf22c86cedc5f7b037c5901886c109d869aa64d1eb82ca70c4960ba73cc2e655bd73867f80a
-  data.tar.gz: 0b29d4b9a641de7a7056785057379e9c26a4709cec7597aa2ee2135f22cc14b52dfd66c78b5ad3204848d775b7e3b3e0537ca34c21defb29155e8dd2a20d8342
+  metadata.gz: c0de4abdc501a6dba9b716e7ad9a15f66985bec98b2978c9f1b25ed0f069d02d8a172f8b78c442e8f90e32098ad1622a3ba16a890c2ed57d896c4c2b427c5c3e
+  data.tar.gz: ed0f32387285db507797e7b34962a20e4dc6174e7389ffa2488afb8724668c11d1e0089e9e2b62a958875e0e2bb9d7f4976d5d0ef887cd83604acb4939d25b7a

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2025-04-03 06:48:36 UTC using RuboCop version 1.75.1.
+# on 2025-07-05 22:42:28 UTC using RuboCop version 1.77.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -13,43 +13,32 @@ Gemspec/DuplicatedAssignment:
   Exclude:
     - 'suma.gemspec'
 
-# Offense count: 2
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
-# SupportedHashRocketStyles: key, separator, table
-# SupportedColonStyles: key, separator, table
-# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
-Layout/HashAlignment:
-  Exclude:
-    - 'lib/suma/cli/validate.rb'
-
-# Offense count: 55
+# Offense count: 57
 # This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
+# Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
 Layout/LineLength:
   Exclude:
     - 'lib/suma/cli.rb'
     - 'lib/suma/cli/build.rb'
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
-    - 'lib/suma/collection_manifest.rb'
     - 'lib/suma/processor.rb'
     - 'lib/suma/schema_attachment.rb'
     - 'lib/suma/schema_collection.rb'
     - 'lib/suma/schema_document.rb'
     - 'lib/suma/thor_ext.rb'
+    - 'spec/suma/cli/extract_terms_spec.rb'
     - 'spec/suma/cli/validate_ascii_spec.rb'
     - 'suma.gemspec'
 
-# Offense count: 3
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AllowInHeredoc.
-Layout/TrailingWhitespace:
+# Offense count: 1
+# Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
+Lint/DuplicateBranch:
   Exclude:
-    - 'lib/suma/cli.rb'
-    - 'lib/suma/cli/validate.rb'
+    - 'lib/suma/cli/extract_terms.rb'
 
 # Offense count: 2
 Lint/DuplicateMethods:
@@ -57,10 +46,11 @@ Lint/DuplicateMethods:
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/express_schema.rb'
 
-# Offense count: 15
+# Offense count: 20
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Exclude:
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/schema_attachment.rb'
@@ -73,15 +63,16 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 64
 
-# Offense count: 4
+# Offense count: 8
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/CyclomaticComplexity:
   Exclude:
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/thor_ext.rb'
 
-# Offense count: 21
+# Offense count: 30
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 107
@@ -91,10 +82,11 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Max: 6
 
-# Offense count: 2
+# Offense count: 5
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/PerceivedComplexity:
   Exclude:
+    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
 
@@ -113,12 +105,12 @@ Performance/CollectionLiteralInLoop:
   Exclude:
     - 'spec/suma/cli_spec.rb'
 
-# Offense count: 3
+# Offense count: 9
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
-  Max: 16
+  Max: 44
 
-# Offense count: 5
+# Offense count: 6
 RSpec/MultipleExpectations:
   Max: 12
 
@@ -136,3 +128,9 @@ Style/EmptyElse:
 # SupportedStyles: annotated, template, unannotated
 Style/FormatStringToken:
   EnforcedStyle: unannotated
+
+# Offense count: 1
+# Configuration parameters: Max.
+Style/SafeNavigationChainLength:
+  Exclude:
+    - 'lib/suma/cli/extract_terms.rb'

data/README.adoc

@@ -37,10 +37,12 @@ $ gem install suma
 # Defaults to `suma help`
 $ suma
 Commands:
-  suma build METANORMA_SITE_MANIFEST      # Build collection specified in site manifest (`metanorma*.yml`)
-  suma reformat EXPRESS_FILE_PATH         # Reformat EXPRESS files
-  suma validate SUBCOMMAND ...ARGS        # Validate express documents
-  suma help [COMMAND]                     # Describe available commands or one specific command
+  suma build METANORMA_SITE_MANIFEST        # Build collection specified in site manifest (`metanorma*.yml`)
+  suma reformat EXPRESS_FILE_PATH           # Reformat EXPRESS files
+  suma validate SUBCOMMAND ...ARGS          # Validate express documents
+  suma generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE # Generate schemas manifest file from Metanorma manifest YAML file
+  suma extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH # Extract terms from schema manifest file
+  suma help [COMMAND]                       # Describe available commands or one specific command
 ----
 
 === Build command
@@ -287,11 +289,134 @@ Replacement: AsciiMath: xx
 ----
 
 
+=== Generate schemas command
+
+The `suma generate_schemas` command generates a schema manifest file containing
+all schemas defined in the Metanorma manifest file.
+
+[source,sh]
+----
+$ suma generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE [options]
+----
+
+Parameters:
+
+`METANORMA_MANIFEST_FILE`:: Path to the Metanorma manifest file
+(e.g.: "metanorma-smrl-all.yml")
+
+Options:
+
+`--exclude_path`, `-e`:: Exclude schemas by pattern (e.g. `*_lf.exp`)
+
+[example]
+====
+.To generate schemas manifest file from Metanorma manifest file
+[source,sh]
+----
+$ bundle exec suma generate_schemas metanorma-smrl-all.yml schemas-smrl-all.yml
+# => generates schemas-smrl-all.yml
+----
+
+.To generate schemas manifest file from Metanorma manifest file and exclude schemas with names like `*_lf.exp`
+[source,sh]
+----
+$ bundle exec suma generate_schemas metanorma-smrl-all.yml schemas-smrl-all.yml -e *_lf.exp
+# => generates schemas-smrl-all.yml without schemas with names like *_lf.exp
+----
+====
+
+All documents need to have a `schemas.yaml` in their document root that lists
+out which schemas the document includes.
+
+
+=== Extract terms command
+
+The `suma extract_terms` command extracts terms from EXPRESS schemas and
+generates a Glossarist v2 dataset in the output directory. This command processes
+various types of STEP schemas and creates standardized terminology datasets
+suitable for glossary and dictionary applications.
+
+[source,sh]
+----
+$ suma extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH [options]
+----
+
+Parameters:
+
+`SCHEMA_MANIFEST_FILE`:: Path to the schema manifest file that lists all schemas
+to process (e.g., "schemas-smrl-all.yml")
+
+`GLOSSARIST_OUTPUT_PATH`:: Path to the output directory where the Glossarist v2
+dataset will be generated
+
+Options:
+
+`--language_code`, `-l`:: Language code for the Glossarist dataset (default: "eng")
+
+==== Supported schema types
+
+The command supports extraction from the following EXPRESS schema types:
+
+* **ARM (Application Reference Model)** - Application module schemas ending with `_arm`
+* **MIM (Module Implementation Model)** - Application module schemas ending with `_mim`
+* **Resource schemas** - General resource schemas
+* **BOM (Business Object Model)** - Business object model schemas ending with `_bom`
+
+==== Extracted terms
+
+The command extracts the following types of terms from EXPRESS schemas:
+
+* **Entities** - EXPRESS entity definitions with their attributes and relationships
+* **Types** - EXPRESS type definitions including enumerations and select types
+* **Functions** - EXPRESS function definitions
+* **Procedures** - EXPRESS procedure definitions
+* **Constants** - EXPRESS constant definitions
+
+Each extracted term includes:
+
+* Unique identifier based on schema and term name
+* Term definition and description
+* Source schema information
+* Appropriate domain classification (application module, resource, or business object model)
+
+==== Output format
+
+The command generates a Glossarist v2 compliant dataset with:
+
+* `concept/` directory containing concept definition files
+* `localized_concept/` directory containing localized concept files
+* YAML format following Glossarist v2 schema specifications
+* Proper cross-references and citations to source schemas
+
+.To extract terms from a schema manifest file
+[example]
+====
+[source,sh]
+----
+$ bundle exec suma extract_terms schemas-smrl-all.yml glossarist_output
+# => generates glossarist_output/concept/*.yaml and glossarist_output/localized_concept/*.yaml
+----
+====
+
+.To extract terms from a specific schema subset
+[example]
+====
+[source,sh]
+----
+$ bundle exec suma extract_terms schemas-activity-modules.yml terms_output
+# => processes only schemas listed in the manifest file
+----
+====
+
+The generated dataset is meant to be used for ISO 10303-2.
+
+
 == Usage: Ruby
 
 === General
 
-Suma can be used programmatically in your Ruby applications. The following examples demonstrate common usage patterns.
+Suma can be used programmatically in your Ruby applications. The following
+examples demonstrate common usage patterns.
 
 === Building collections
 

data/lib/suma/cli/extract_terms.rb

@@ -0,0 +1,443 @@
+# frozen_string_literal: true
+
+require "thor"
+require_relative "../thor_ext"
+require "fileutils"
+require "expressir"
+require "securerandom"
+require "glossarist"
+
+module Suma
+  module Cli
+    # ExtractTerms command using Expressir to extract terms into the
+    # Glossarist v2 format
+    class ExtractTerms < Thor
+      desc "extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
+           "Extract terms from SCHEMA_MANIFEST_FILE into " \
+           "Glossarist v2 format"
+      option :language_code, type: :string, default: "eng", aliases: "-l",
+                             desc: "Language code for the Glossarist"
+
+      def extract_terms(schema_manifest_file, output_path)
+        language_code = options[:language_code]
+        schema_manifest_file = File.expand_path(schema_manifest_file)
+
+        unless File.exist?(schema_manifest_file)
+          raise Errno::ENOENT, "Specified SCHEMA_MANIFEST_FILE " \
+                               "`#{schema_manifest_file}` not found."
+        end
+
+        run(schema_manifest_file, output_path, language_code)
+      end
+
+      private
+
+      def run(schema_manifest_file, output_path, language_code = "eng")
+        get_exp_files(schema_manifest_file).map do |exp_file|
+          extract(exp_file, output_path, language_code)
+        end
+      end
+
+      def get_exp_files(schema_manifest_file)
+        config = Expressir::SchemaManifest.from_file(schema_manifest_file)
+        paths = config.schemas.map(&:path)
+
+        if paths.empty?
+          raise Errno::ENOENT, "No EXPRESS files found in " \
+                               "`#{schema_manifest_file}`."
+        end
+
+        paths
+      end
+
+      def extract(exp_file, output_path, language_code)
+        exp_path_rel = Pathname.new(exp_file).relative_path_from(Pathname.getwd)
+        puts "Building terms: #{exp_path_rel}"
+
+        repo = Expressir::Express::Parser.from_file(exp_file)
+        schema = get_default_schema(repo)
+
+        unless schema.file
+          raise Error.new("Schema must have an associated file")
+        end
+
+        collection = build_managed_concept_collection(
+          schema, language_code
+        )
+
+        output_data(collection, output_path)
+      end
+
+      def output_data(collection, output_path)
+        unless File.exist?(output_path)
+          FileUtils.mkdir_p(File.expand_path(output_path))
+        end
+
+        puts "Saving collection to files in: #{output_path}"
+        collection.save_to_files(File.expand_path(output_path))
+
+        collection
+      end
+
+      def build_managed_concept_collection(schema, language_code)
+        Glossarist::ManagedConceptCollection.new.tap do |collection|
+          # Extract schema-level citation data once to reuse across all entities
+          source_ref = get_source_ref(schema)
+
+          # Create one concept per entity
+          schema.entities.each do |entity|
+            localized_concept = build_localized_concept(
+              schema: schema,
+              entity: entity,
+              language_code: language_code,
+              source_ref: source_ref,
+            )
+
+            managed_data = Glossarist::ManagedConceptData.new.tap do |data|
+              data.id = get_entity_identifier(schema, entity)
+
+              # TODO: Why do we need both localizations and localized_concepts??
+              data.localizations[language_code] = localized_concept
+              # uuid is automatically set from the serialization of the object
+              data.localized_concepts = {
+                language_code => get_localized_concept_identifier(
+                  schema, entity, language_code
+                ),
+              }
+            end
+
+            managed_concept = Glossarist::ManagedConcept.new.tap do |concept|
+              # uuid is automatically set from the serialization of the object
+              concept.id = get_entity_identifier(schema, entity)
+              concept.uuid = concept.id
+              concept.data = managed_data
+            end
+
+            collection.store(managed_concept)
+          end
+        end
+      end
+
+      def build_localized_concept(schema:, entity:, language_code:, source_ref:)
+        schema_domain = get_domain(schema)
+
+        localized_concept_data = Glossarist::ConceptData.new.tap do |data|
+          data.terms = get_entity_terms(entity)
+          data.definition = get_entity_definitions(entity, schema)
+          data.language_code = language_code
+          data.domain = schema_domain
+          data.sources = [source_ref] if source_ref
+
+          # Only assign optional fields if they have content
+          notes = get_entity_notes(entity, schema_domain)
+          data.notes = notes if notes && !notes.empty?
+
+          examples = get_entity_examples(entity, schema_domain)
+          data.examples = examples if examples && !examples.empty?
+        end
+
+        Glossarist::LocalizedConcept.new.tap do |concept|
+          concept.data = localized_concept_data
+        end
+      end
+
+      # We only deal with 1 schema
+      def get_default_schema(repo)
+        repo.schemas.first
+      end
+
+      def find_remark_value(schema, remark_id)
+        schema.remark_items.find { |s| s.id == remark_id }&.remarks&.first
+      end
+
+      def get_entity_identifier(schema, entity)
+        "#{schema.id}.#{entity.id}"
+      end
+
+      def get_localized_concept_identifier(schema, entity, lang)
+        "#{schema.id}.#{entity.id}-#{lang}"
+      end
+
+      def get_source_ref(schema)
+        origin = Glossarist::Citation.new.tap do |citation|
+          citation.ref = "ISO 10303"
+          custom_locality = build_custom_locality(schema)
+
+          unless custom_locality.empty?
+            citation.custom_locality = custom_locality
+          end
+        end
+
+        Glossarist::ConceptSource.new(type: "authoritative", origin: origin)
+      end
+
+      # SCHEMA action_schema
+      # '{iso standard 10303 part(41) version(9) object(1) action-schema(1)}';
+      def build_custom_locality(schema)
+        [].tap do |localities|
+          # Add schema name
+          localities << Glossarist::CustomLocality.new(
+            name: "schema",
+            value: schema.id,
+          )
+
+          # Add version if available
+          version_item = schema.version.items.detect { |i| i.name == "version" }
+          if version_item
+            localities << Glossarist::CustomLocality.new(
+              name: "version",
+              value: version_item.value,
+            )
+          end
+        end
+      end
+
+      # TODO: What if this was a "bom"?
+      def get_domain(schema)
+        prefix = if mim?(schema.id) || arm?(schema.id)
+                   "application module"
+                 else
+                   "resource"
+                 end
+
+        "#{prefix}: #{schema.id}"
+      end
+
+      def arm?(schema_id)
+        schema_id.end_with?("_arm")
+      end
+
+      def mim?(schema_id)
+        schema_id.end_with?("_mim")
+      end
+
+      def get_terms(schema)
+        schema_title = get_title(schema)
+        if schema_title
+          [
+            Glossarist::Designation::Base.new(
+              designation: schema_title,
+              type: "expression",
+              normative_status: "preferred",
+            ),
+          ]
+        end
+      end
+
+      def get_entity_terms(entity)
+        # For now, use the entity ID as the term
+        # This could be enhanced to look for entity-specific title remark items
+        [
+          Glossarist::Designation::Base.new(
+            designation: entity.id,
+            type: "expression",
+            normative_status: "preferred",
+          ),
+        ]
+      end
+
+      def get_entity_definitions(entity, schema)
+        schema_type = extract_file_type(schema.file)
+        schema_domain = get_domain(schema)
+
+        definition = generate_entity_definition(entity, schema_domain,
+                                                schema_type)
+        [Glossarist::DetailedDefinition.new(content: definition)]
+      end
+
+      def get_entity_notes(entity, schema_domain)
+        notes = []
+
+        # Add trimmed definition from entity description as first note
+        if entity.remarks && !entity.remarks.empty?
+          trimmed_def = trim_definition(entity.remarks)
+          if trimmed_def && !trimmed_def.empty?
+            notes << Glossarist::DetailedDefinition.new(
+              content: convert_express_xref(trimmed_def, schema_domain),
+            )
+          end
+        end
+
+        # Add other notes
+        other_notes = [
+          entity.remark_items&.select do |ri|
+            ri.id == "__note"
+          end&.map(&:remarks),
+        ].flatten.compact
+
+        other_notes.each do |note|
+          notes << Glossarist::DetailedDefinition.new(
+            content: convert_express_xref(note, schema_domain),
+          )
+        end
+
+        notes
+      end
+
+      def get_entity_examples(entity, schema_domain)
+        examples = entity.remark_items&.select do |ri|
+          ri.id == "__example"
+        end&.map(&:remarks)&.flatten&.compact || []
+
+        examples.map do |example|
+          Glossarist::DetailedDefinition.new(
+            content: convert_express_xref(example, schema_domain),
+          )
+        end
+      end
+
+      def extract_file_type(filename)
+        match = filename.match(/(arm|mim|bom)_annotated\.exp$/)
+        return "resource" unless match
+
+        {
+          "arm" => "module_arm",
+          "mim" => "module_mim",
+          "bom" => "business_object_model",
+        }[match.captures[0]] || "resource"
+      end
+
+      def get_schema_type(schema)
+        return "mim" if mim?(schema.id)
+        return "arm" if arm?(schema.id)
+        return "bom" if bom?(schema.id)
+
+        "resource"
+      end
+
+      def bom?(schema_id)
+        schema_id.end_with?("_bom")
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def combine_paragraphs(full_paragraph, next_paragraph)
+        # If full_paragraph already contains a period, extract that.
+        if m = full_paragraph.match(/\A(?<inner_first>[^\n]*?\.)\s/)
+          # puts "CONDITION 1"
+          if m[:inner_first]
+            return m[:inner_first]
+          else
+            return full_paragraph
+          end
+        end
+
+        # If full_paragraph ends with a period, this is the last.
+        if /\.\s*\Z/.match?(full_paragraph)
+          # puts "CONDITION 2"
+          return full_paragraph
+        end
+
+        # If next_paragraph is a list
+        if next_paragraph.start_with?("*")
+          # puts "CONDITION 3"
+          return "#{full_paragraph}\n\n#{next_paragraph}"
+        end
+
+        # If next_paragraph is a continuation of a list
+        if next_paragraph.start_with?("which", "that")
+          # puts "CONDITION 4"
+          return "#{full_paragraph}\n\n#{next_paragraph}"
+        end
+
+        # puts "CONDITION 5"
+        full_paragraph
+      end
+
+      def trim_definition(definition)
+        return nil if definition.nil? || definition.empty?
+
+        # Handle case where definition is an array
+        definition_str = if definition.is_a?(Array)
+                           definition.join("\n\n")
+                         else
+                           definition.to_s
+                         end
+
+        return nil if definition_str.empty?
+
+        # Unless the first paragraph ends with "between" and is followed by a
+        # list, don't split
+        paragraphs = definition_str.split("\n\n")
+
+        # puts paragraphs.inspect
+
+        first_paragraph = paragraphs.first
+
+        combined = if paragraphs.length > 1
+                     paragraphs[1..].inject(first_paragraph) do |acc, p|
+                       combine_paragraphs(acc, p)
+                     end
+                   else
+                     combine_paragraphs(first_paragraph, "")
+                   end
+
+        # puts "combined--------- #{combined}"
+
+        # Remove comments until end of line
+        combined = "#{combined}\n"
+        combined.gsub!(/\n\/\/.*?\n/, "\n")
+        combined.strip!
+
+        express_reference_to_mention(combined)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # Replace `<<express:{schema}.{entity},{render}>>` with {{entity,render}}
+      def express_reference_to_mention(description)
+        # TODO: Use Expressir to check whether the "entity" is really an
+        # EXPRESS ENTITY. If not, skip the mention.
+        description.gsub(/<<express:([^,]+),([^>]+)>>/) do |_match|
+          "{{#{Regexp.last_match[1].split('.').last},#{Regexp.last_match[2]}}}"
+        end
+      end
+
+      def entity_name_to_text(entity_id)
+        entity_id.downcase.gsub("_", " ")
+      end
+
+      # rubocop:disable Layout/LineLength
+      def generate_entity_definition(entity, _domain, schema_type)
+        return "" if entity.nil?
+
+        # See: metanorma/iso-10303-2#90
+        entity_type = case schema_type
+                      when "module_arm"
+                        "{{application object}}"
+                      when "module_mim"
+                        "{{entity data type}}"
+                      when "resource", "business_object_model"
+                        "{{entity data type}}"
+                      else
+                        raise Error.new("[suma] encountered unsupported schema_type")
+                      end
+
+        if entity.subtype_of.empty?
+          "#{entity_type} " \
+            "that represents the " \
+            "#{entity_name_to_text(entity.id)} {{entity}}"
+        else
+          entity_subtypes = entity.subtype_of.map do |e|
+            "{{#{e.id}}}"
+          end
+
+          "#{entity_type} that is a type of " \
+            "#{entity_subtypes.join(' and ')} " \
+            "that represents the " \
+            "#{entity_name_to_text(entity.id)} {{entity}}"
+        end
+      end
+      # rubocop:enable Layout/LineLength
+
+      def convert_express_xref(content, schema_domain)
+        content.gsub(/<<express:(.*),(.*)>>/) do
+          "{{<#{schema_domain}>" \
+            "#{Regexp.last_match(1).split('.').last},#{Regexp.last_match(2)}}}"
+        end
+      end
+
+      def id_from_designation(designation)
+        designation.gsub(" ", "_").gsub("/", "_").gsub(":", "_")
+      end
+    end
+  end
+end

data/lib/suma/cli/generate_schemas.rb

@@ -0,0 +1,141 @@
+require "thor"
+require_relative "../thor_ext"
+require "fileutils"
+require "yaml"
+require "pathname"
+
+module Suma
+  module Cli
+    # GenerateSchemas command to generate Schemas YAML by Metanorma YAML
+    class GenerateSchemas < Thor
+      desc "generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE",
+           "Generate EXPRESS schema manifest file from Metanorma site manifest"
+      option :exclude_paths, type: :string, default: nil, aliases: "-e",
+                             desc: "Exclude schemas paths by pattern " \
+                                   "(e.g. `*_lf.exp`)"
+
+      YAML_FILE_EXTENSIONS = [".yaml", ".yml"].freeze
+
+      def generate_schemas(metanorma_manifest_file, schema_manifest_file) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        metanorma_manifest_file = File.expand_path(metanorma_manifest_file)
+
+        unless File.exist?(metanorma_manifest_file)
+          raise Errno::ENOENT, "Specified file `#{metanorma_manifest_file}` " \
+                               "not found."
+        end
+
+        unless File.file?(metanorma_manifest_file)
+          raise ArgumentError, "Specified path `#{metanorma_manifest_file}` " \
+                               "is not a file."
+        end
+
+        [metanorma_manifest_file, schema_manifest_file].each do |file|
+          if !YAML_FILE_EXTENSIONS.include?(File.extname(file))
+            raise ArgumentError, "Specified file `#{file}` is not a YAML file."
+          end
+        end
+
+        run(
+          metanorma_manifest_file, schema_manifest_file,
+          exclude_paths: options[:exclude_paths]
+        )
+      end
+
+      private
+
+      def run(metanorma_manifest_file, schema_manifest_file, exclude_paths: nil)
+        metanorma_data = load_yaml(metanorma_manifest_file)
+        collection_files = metanorma_data["metanorma"]["source"]["files"]
+        manifest_files = load_manifest_files(collection_files)
+        all_schemas = load_project_schemas(manifest_files, exclude_paths,
+                                           schema_manifest_file)
+        all_schemas["schemas"] = all_schemas["schemas"].sort.to_h
+        output_data(all_schemas, schema_manifest_file)
+      end
+
+      def output_data(all_schemas, path)
+        puts "Writing the Schemas YAML file to #{File.expand_path(path)}..."
+        # debug use only
+        # puts all_schemas.to_yaml
+        File.write(File.expand_path(path), all_schemas.to_yaml)
+        puts "Writing the Schemas YAML file to #{File.expand_path(path)}...Done"
+      end
+
+      def load_yaml(file_path)
+        YAML.safe_load(
+          File.read(file_path, encoding: "UTF-8"),
+          permitted_classes: [Date, Time, Symbol],
+          permitted_symbols: [],
+          aliases: true,
+        )
+      end
+
+      def load_manifest_files(collection_files)
+        manifest_files = collection_files.map do |c|
+          collection_data = load_yaml(c)
+          collection_data["manifest"]["docref"].map { |docref| docref["file"] }
+        end
+        manifest_files.flatten
+      end
+
+      def load_project_schemas( # rubocop:disable Metrics/AbcSize
+        manifest_files, exclude_paths, schema_manifest_file
+      )
+        all_schemas = { "schemas" => {} }
+
+        manifest_files.each do |file|
+          # load schemas.yaml from the location of the collection.yml file
+          schemas_file_path = File.expand_path(
+            file.gsub("collection.yml", "schemas.yaml"),
+          )
+          unless File.exist?(schemas_file_path)
+            puts "Schemas file not found: #{schemas_file_path}"
+            next
+          end
+
+          schemas_data = load_yaml(schemas_file_path)
+
+          if schemas_data["schemas"]
+            schemas_data["schemas"] = fix_path(
+              schemas_data,
+              schemas_file_path,
+              schema_manifest_file,
+            )
+            all_schemas["schemas"].merge!(schemas_data["schemas"])
+          end
+
+          if exclude_paths
+            all_schemas["schemas"].delete_if do |_key, value|
+              value["path"].match?(
+                Regexp.new(exclude_paths.gsub("*", "(.*){1,999}")),
+              )
+            end
+          end
+        end
+
+        all_schemas
+      end
+
+      def fix_path(schemas_data, schemas_file_path, schema_manifest_file) # rubocop:disable Metrics/AbcSize
+        schema_manifest_path = File.expand_path(schema_manifest_file, Dir.pwd)
+
+        schemas_data["schemas"].each do |key, value|
+          # resolve the path in the schema file by the path in the schemas.yaml
+          path_in_schema = File.expand_path(
+            value["path"],
+            File.dirname(schemas_file_path),
+          )
+
+          # calculate the relative path from the schema manifest file
+          fixed_path = Pathname.new(path_in_schema).relative_path_from(
+            Pathname.new(File.dirname(schema_manifest_path)),
+          )
+
+          { key => value.merge!("path" => fixed_path.to_s) }
+        end
+
+        schemas_data["schemas"]
+      end
+    end
+  end
+end

data/lib/suma/cli/reformat.rb

@@ -57,7 +57,7 @@ module Suma
         # containing '(*text*)' inside
         comments = file_content.scan(/\(\*"(.*?)\n\*\)/m).map(&:first)
 
-        if comments.count.positive?
+        if comments.any?
           content_without_comments = file_content.gsub(/\(\*".*?\n\*\)/m, "")
 
           # remove extra newlines

data/lib/suma/cli/validate_links.rb

@@ -2,6 +2,7 @@
 
 require "thor"
 require_relative "../utils"
+require "expressir"
 
 module Suma
   module Cli
@@ -43,7 +44,6 @@ module Suma
         # Lazy-load dependencies only when this command is actually used
         require "expressir"
         require "ruby-progressbar"
-        require_relative "../schema_config"
         require "pathname"
       end
 
@@ -74,7 +74,7 @@ module Suma
 
       # Load and initialize the schemas configuration
       def load_schemas_config(schemas_file_path)
-        schemas_config = Suma::SchemaConfig::Config.from_yaml(File.read(schemas_file_path))
+        schemas_config = Expressir::SchemaManifest.from_yaml(File.read(schemas_file_path))
         # Ensure the config is initialized with the correct path to resolve relative paths
         schemas_config.set_initial_path(schemas_file_path.to_s)
         schemas_config

data/lib/suma/cli.rb

@@ -23,6 +23,16 @@ module Suma
         Cli::Build.start
       end
 
+      desc "generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE",
+           "Generate EXPRESS schema manifest file from Metanorma site manifest"
+      option :exclude_paths, type: :string, default: nil, aliases: "-e",
+                             desc: "Exclude schemas paths by pattern " \
+                                   "(e.g. `*_lf.exp`)"
+      def generate_schemas(_metanorma_manifest_file, _schema_manifest_file)
+        require_relative "cli/generate_schemas"
+        Cli::GenerateSchemas.start
+      end
+
       desc "reformat EXPRESS_FILE_PATH",
            "Reformat EXPRESS files"
       option :recursive, type: :boolean, default: false, aliases: "-r",
@@ -33,6 +43,16 @@ module Suma
         Cli::Reformat.start
       end
 
+      desc "extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
+           "Extract terms from SCHEMA_MANIFEST_FILE into " \
+           "Glossarist v2 format"
+      option :language_code, type: :string, default: "eng", aliases: "-l",
+                             desc: "Language code for the Glossarist"
+      def extract_terms(_schema_manifest_file, _glossarist_output_path)
+        require_relative "cli/extract_terms"
+        Cli::ExtractTerms.start
+      end
+
       desc "validate SUBCOMMAND ...ARGS", "Validate express documents"
       subcommand "validate", Cli::Validate
 

data/lib/suma/collection_manifest.rb

@@ -3,6 +3,7 @@
 require "metanorma/cli"
 require "metanorma/cli/collection"
 require "metanorma/collection/collection"
+require "expressir"
 
 module Suma
   class CollectionManifest < Metanorma::Collection::Config::Manifest
@@ -36,7 +37,7 @@ module Suma
     end
 
     def export_schema_config(path)
-      export_config = @schema_config || Suma::SchemaConfig::Config.new
+      export_config = @schema_config || Expressir::SchemaManifest.new
       return export_config unless entry
 
       entry.each do |x|
@@ -127,7 +128,7 @@ module Suma
       if File.basename(file) == "collection.yml"
         schemas_yaml_path = File.join(File.dirname(file), "schemas.yaml")
         if schemas_yaml_path && File.exist?(schemas_yaml_path)
-          @schema_config = Suma::SchemaConfig::Config.from_file(schemas_yaml_path)
+          @schema_config = Expressir::SchemaManifest.from_file(schemas_yaml_path)
         end
       end
     end

data/lib/suma/processor.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "schema_config"
 require_relative "schema_collection"
 require_relative "utils"
 require_relative "collection_config"

data/lib/suma/schema_attachment.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require "expressir"
 # require "metanorma/cli"
-require_relative "schema_config"
 
 module Suma
   class SchemaAttachment
@@ -68,8 +68,8 @@ module Suma
 
     def to_config(path: nil)
       # return @config unless @config
-      @config = SchemaConfig::Config.new
-      @config.schemas << SchemaConfig::Schema.new(
+      @config = Expressir::SchemaManifest.new
+      @config.schemas << Expressir::SchemaManifestEntry.new(
         id: @schema.id,
         path: @schema.path,
       )

data/lib/suma/schema_collection.rb

@@ -3,7 +3,7 @@
 require_relative "express_schema"
 require_relative "schema_attachment"
 require_relative "schema_document"
-require_relative "schema_config"
+require "expressir"
 require_relative "utils"
 
 module Suma
@@ -19,7 +19,7 @@ module Suma
       @output_path_docs = Pathname.new(output_path_docs || Dir.pwd).expand_path
       @output_path_schemas = Pathname.new(output_path_schemas || Dir.pwd).expand_path
       @config = config
-      @config ||= config_yaml && SchemaConfig::Config.from_file(config_yaml)
+      @config ||= config_yaml && Expressir::SchemaManifest.from_file(config_yaml)
       @manifest = manifest
     end
 

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.1.13"
+  VERSION = "0.1.15"
 end

data/lib/suma.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "expressir"
 require_relative "suma/version"
 require_relative "suma/processor"
 

data/suma.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.require_paths = ["lib"]
 
   spec.add_dependency "expressir", "~> 2.1"
+  spec.add_dependency "glossarist", "~> 2.3.7"
   spec.add_dependency "lutaml-model", "~> 0.7"
   spec.add_dependency "metanorma-cli"
   spec.add_dependency "plurimath"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.15
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-05-24 00:00:00.000000000 Z
+date: 2025-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: glossarist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.7
 - !ruby/object:Gem::Dependency
   name: lutaml-model
   requirement: !ruby/object:Gem::Requirement
@@ -134,6 +148,8 @@ files:
 - lib/suma.rb
 - lib/suma/cli.rb
 - lib/suma/cli/build.rb
+- lib/suma/cli/extract_terms.rb
+- lib/suma/cli/generate_schemas.rb
 - lib/suma/cli/reformat.rb
 - lib/suma/cli/validate.rb
 - lib/suma/cli/validate_ascii.rb
@@ -144,9 +160,6 @@ files:
 - lib/suma/processor.rb
 - lib/suma/schema_attachment.rb
 - lib/suma/schema_collection.rb
-- lib/suma/schema_config.rb
-- lib/suma/schema_config/config.rb
-- lib/suma/schema_config/schema.rb
 - lib/suma/schema_document.rb
 - lib/suma/site_config.rb
 - lib/suma/thor_ext.rb

data/lib/suma/schema_config/config.rb

@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-require "lutaml/model"
-require_relative "schema"
-require_relative "../utils"
-
-module Suma
-  module SchemaConfig
-    class Config < Lutaml::Model::Serializable
-      attribute :schemas, Schema, collection: true, initialize_empty: true
-      attribute :path, Lutaml::Model::Type::String
-      attr_accessor :output_path
-
-      def initialize(**args)
-        @path = path_relative_to_absolute(path) if path
-        super
-      end
-
-      def base_path
-        File.dirname(@path)
-      end
-
-      yaml do
-        map "schemas", with: { from: :schemas_from_yaml, to: :schemas_to_yaml }
-      end
-
-      def self.from_file(path)
-        from_yaml(File.read(path)).tap do |x|
-          x.set_initial_path(path)
-        end
-      end
-
-      def to_file(to_path = path)
-        File.write(to_path, to_yaml)
-      end
-
-      def set_initial_path(new_path)
-        @path = path_relative_to_absolute(new_path)
-        schemas.each do |schema|
-          schema.path = path_relative_to_absolute(schema.path)
-          schema.container_path = File.expand_path(@path)
-        end
-      end
-
-      def schemas_from_yaml(model, value)
-        model.schemas = value.map do |k, v|
-          Schema.new(id: k, path: path_relative_to_absolute(v["path"]))
-        end
-      end
-
-      def schemas_to_yaml(model, doc)
-        doc["schemas"] = model.schemas.sort_by(&:id).to_h do |schema|
-          # We are outputting the schemas collection file to the directory where
-          # the collection config is at (assumed to be Dir.pwd), not to the
-          # directory we sourced the manifest from, e.g.
-          # documents/iso-10303-41/schemas.yaml.
-
-          # So the schema.container_path = @config.path is not
-          # in fact needed, as the files are already absolute. This notion of
-          # using @path to create relative paths was misconceived
-          [
-            schema.id,
-            {
-              "path" => path_absolute_to_relative(
-                schema.path,
-                model.output_path || Dir.pwd,
-              ),
-            },
-          ]
-        end
-      end
-
-      def path_relative_to_absolute(relative_path)
-        eval_path = Pathname.new(relative_path)
-        return relative_path if eval_path.absolute?
-
-        # Or based on current working directory?
-        return relative_path unless @path
-
-        # ... but if this calculates path, we end up expanding it anyway
-
-        Pathname.new(File.dirname(@path)).join(eval_path).expand_path.to_s
-      end
-
-      def path_absolute_to_relative(absolute_path, container_path)
-        container_path ||= @path
-        return absolute_path unless container_path
-
-        p = Pathname.new(container_path)
-        container = p.directory? ? p.to_s : p.dirname
-        Pathname.new(absolute_path).relative_path_from(container).to_s
-      end
-
-      def update_path(new_path)
-        if @path.nil?
-          @path = new_path
-          return @path
-        end
-
-        old_base_path = File.dirname(@path)
-        new_base_path = File.dirname(new_path)
-        update_schema_path(old_base_path, new_base_path)
-
-        @path = new_path
-      end
-
-      def concat(another_config)
-        unless another_config.is_a?(self.class)
-          raise StandardError, "Can only concat a SchemaConfig::Config object."
-        end
-
-        # We need to update the relative paths when paths exist
-        if path && another_config.path && path != another_config.path
-          new_config = another_config.dup
-          new_config.update_path(path)
-        end
-
-        schemas.concat(another_config.schemas)
-      end
-
-      def save_to_path(filename)
-        new_config = dup.tap do |c|
-          c.path = filename
-          c.update_path(File.dirname(filename))
-          c.output_path = filename
-        end
-
-        File.open(filename, "w") do |f|
-          Utils.log "Writing #{filename}..."
-          f.write(new_config.to_yaml)
-          Utils.log "Done."
-        end
-      end
-
-      def update_schema_path(old_base_path, new_base_path)
-        schemas.each do |schema|
-          schema_path = Pathname.new(schema.path)
-          next if schema_path.absolute?
-
-          schema_path = (Pathname.new(old_base_path) + schema_path).cleanpath
-          # This is the new relative schema_path
-          schema.path = schema_path.relative_path_from(new_base_path)
-        end
-      end
-    end
-  end
-end

data/lib/suma/schema_config/schema.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-require "lutaml/model"
-
-module Suma
-  module SchemaConfig
-    class Schema < Lutaml::Model::Serializable
-      attribute :id, Lutaml::Model::Type::String
-      attribute :path, Lutaml::Model::Type::String
-      # attribute :schemas_only, Lutaml::Model::Type::Boolean
-
-      # container_path is a copy of Suma::SchemaConfig::Config.path,
-      # used to resolve the path of each schema within
-      # Suma::SchemaConfig::Config.schemas,
-      # when Suma::SchemaConfig::Config.schemas is recursively flattened
-      attr_accessor :container_path
-    end
-  end
-end

data/lib/suma/schema_config.rb

@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "schema_config/schema"
-require_relative "schema_config/config"