suma 0.1.15 → 0.1.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52836adb9e003f9d760a6c015207379be88b8821855d64388a5f30d09474de7e
-  data.tar.gz: d7427de3e6dc21291671122bd07bc23c739e4f59f031e3b02988f89254767d00
+  metadata.gz: 5ecace379e8cbe19d7f3d6144ce5862be6529c02b418352c79b382f0be184e67
+  data.tar.gz: c1ce481cd14a29ce112782a088c8b11722c7d091e17df3ca771339c0446b1396
 SHA512:
-  metadata.gz: c0de4abdc501a6dba9b716e7ad9a15f66985bec98b2978c9f1b25ed0f069d02d8a172f8b78c442e8f90e32098ad1622a3ba16a890c2ed57d896c4c2b427c5c3e
-  data.tar.gz: ed0f32387285db507797e7b34962a20e4dc6174e7389ffa2488afb8724668c11d1e0089e9e2b62a958875e0e2bb9d7f4976d5d0ef887cd83604acb4939d25b7a
+  metadata.gz: 4c84af1274e4b93530d213648de1a32cd580203ca8da2123f2071a6457c007b87ed343fbe8326ac652cd706c650603c959aea0ebb54d2f5109a6e92098e5ccd7
+  data.tar.gz: 4b1ee08b731c160b772a89c9dd5af33c7720fcc388bf8b8e9cf06554a4c07c8f052cf78f953f4841a48543ef255626e666b9f4c6bd0a4e225a956684a82954bc

data/.gitignore

@@ -9,3 +9,7 @@
 
 # rspec failure tracking
 .rspec_status
+.rubocop_todo.yml
+.rubocop-https---*-yml
+.ruby-version
+Gemfile.lock

data/.rubocop.yml

@@ -8,7 +8,7 @@ plugins:
 - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 3.0
+  TargetRubyVersion: 3.1
   NewCops: enable
   Exclude:
     - 'vendor/**/*'

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2025-07-05 22:42:28 UTC using RuboCop version 1.77.0.
+# on 2025-07-17 08:10:53 UTC using RuboCop version 1.78.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -13,7 +13,30 @@ Gemspec/DuplicatedAssignment:
   Exclude:
     - 'suma.gemspec'
 
-# Offense count: 57
+# Offense count: 1
+# Configuration parameters: Severity, Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'suma.gemspec'
+
+# Offense count: 17
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: with_first_argument, with_fixed_indentation
+Layout/ArgumentAlignment:
+  Exclude:
+    - 'lib/suma/cli/validate_ascii.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: normal, indented_internal_methods
+Layout/IndentationConsistency:
+  Exclude:
+    - 'lib/suma/glossarist_extensions.rb'
+
+# Offense count: 90
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
@@ -21,10 +44,10 @@ Layout/LineLength:
   Exclude:
     - 'lib/suma/cli.rb'
     - 'lib/suma/cli/build.rb'
-    - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
+    - 'lib/suma/glossarist_extensions.rb'
     - 'lib/suma/processor.rb'
     - 'lib/suma/schema_attachment.rb'
     - 'lib/suma/schema_collection.rb'
@@ -34,6 +57,13 @@ Layout/LineLength:
     - 'spec/suma/cli/validate_ascii_spec.rb'
     - 'suma.gemspec'
 
+# Offense count: 22
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowInHeredoc.
+Layout/TrailingWhitespace:
+  Exclude:
+    - 'lib/suma/cli/validate_ascii.rb'
+
 # Offense count: 1
 # Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches, IgnoreDuplicateElseBranch.
 Lint/DuplicateBranch:
@@ -46,33 +76,48 @@ Lint/DuplicateMethods:
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/express_schema.rb'
 
-# Offense count: 20
+# Offense count: 2
+# This cop supports unsafe autocorrection (--autocorrect-all).
+Lint/NonAtomicFileOperation:
+  Exclude:
+    - 'lib/suma/glossarist_extensions.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AutoCorrect, IgnoreEmptyBlocks, AllowUnusedKeywordArguments.
+Lint/UnusedBlockArgument:
+  Exclude:
+    - 'lib/suma/glossarist_extensions.rb'
+
+# Offense count: 23
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Exclude:
     - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
+    - 'lib/suma/glossarist_extensions.rb'
     - 'lib/suma/schema_attachment.rb'
     - 'lib/suma/schema_document.rb'
     - 'lib/suma/thor_ext.rb'
 
-# Offense count: 1
+# Offense count: 3
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
 # AllowedMethods: refine
 Metrics/BlockLength:
   Max: 64
 
-# Offense count: 8
+# Offense count: 9
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/CyclomaticComplexity:
   Exclude:
     - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
+    - 'lib/suma/glossarist_extensions.rb'
     - 'lib/suma/thor_ext.rb'
 
-# Offense count: 30
+# Offense count: 36
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 107
@@ -82,13 +127,14 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Max: 6
 
-# Offense count: 5
+# Offense count: 6
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/PerceivedComplexity:
   Exclude:
     - 'lib/suma/cli/extract_terms.rb'
     - 'lib/suma/cli/validate_ascii.rb'
     - 'lib/suma/cli/validate_links.rb'
+    - 'lib/suma/glossarist_extensions.rb'
 
 # Offense count: 5
 # Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns.
@@ -114,13 +160,14 @@ RSpec/ExampleLength:
 RSpec/MultipleExpectations:
   Max: 12
 
-# Offense count: 1
+# Offense count: 2
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AutoCorrect, EnforcedStyle, AllowComments.
 # SupportedStyles: empty, nil, both
 Style/EmptyElse:
   Exclude:
     - 'lib/suma/cli/validate_links.rb'
+    - 'lib/suma/glossarist_extensions.rb'
 
 # Offense count: 4
 # This cop supports safe autocorrection (--autocorrect).
@@ -129,8 +176,37 @@ Style/EmptyElse:
 Style/FormatStringToken:
   EnforcedStyle: unannotated
 
+# Offense count: 2
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: AllowedReceivers.
+# AllowedReceivers: Thread.current
+Style/HashEachMethods:
+  Exclude:
+    - 'lib/suma/glossarist_extensions.rb'
+
+# Offense count: 2
+# This cop supports unsafe autocorrection (--autocorrect-all).
+Style/IdenticalConditionalBranches:
+  Exclude:
+    - 'lib/suma/glossarist_extensions.rb'
+
+# Offense count: 3
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: ConvertCodeThatCanStartToReturnNil, AllowedMethods, MaxChainLength.
+# AllowedMethods: present?, blank?, presence, try, try!
+Style/SafeNavigation:
+  Exclude:
+    - 'lib/suma/glossarist_extensions.rb'
+
 # Offense count: 1
 # Configuration parameters: Max.
 Style/SafeNavigationChainLength:
   Exclude:
     - 'lib/suma/cli/extract_terms.rb'
+
+# Offense count: 4
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: Mode.
+Style/StringConcatenation:
+  Exclude:
+    - 'lib/suma/cli/validate_ascii.rb'

data/README.adoc

@@ -291,8 +291,9 @@ Replacement: AsciiMath: xx
 
 === Generate schemas command
 
-The `suma generate_schemas` command generates a schema manifest file containing
-all schemas defined in the Metanorma manifest file.
+The `suma generate_schemas` command generates an EXPRESS schema manifest file
+containing all schemas of documents referenced in the Metanorma manifest file,
+recursively.
 
 [source,sh]
 ----
@@ -331,6 +332,10 @@ out which schemas the document includes.
 
 === Extract terms command
 
+The "extract terms" command is implemented for ISO 10303-2, and could also be
+used for other EXPRESS schema collections that require term extraction for
+glossary or dictionary applications.
+
 The `suma extract_terms` command extracts terms from EXPRESS schemas and
 generates a Glossarist v2 dataset in the output directory. This command processes
 various types of STEP schemas and creates standardized terminology datasets
@@ -357,36 +362,35 @@ Options:
 
 The command supports extraction from the following EXPRESS schema types:
 
-* **ARM (Application Reference Model)** - Application module schemas ending with `_arm`
-* **MIM (Module Implementation Model)** - Application module schemas ending with `_mim`
+* **ARM (application reference model)** - application module schemas ending with `_arm`
+* **MIM (module implementation model)** - application module schemas ending with `_mim`
 * **Resource schemas** - General resource schemas
-* **BOM (Business Object Model)** - Business object model schemas ending with `_bom`
+* **BOM (business object model)** - business object model schemas ending with `_bom`
+
+NOTE: The "Long Form" EXPRESS schemas (ending with `_lf`) should be excluded
+from the manifest file as they do not contain definitions of `ENTITY` objects.
 
-==== Extracted terms
 
-The command extracts the following types of terms from EXPRESS schemas:
+==== Extracted concepts
 
-* **Entities** - EXPRESS entity definitions with their attributes and relationships
-* **Types** - EXPRESS type definitions including enumerations and select types
-* **Functions** - EXPRESS function definitions
-* **Procedures** - EXPRESS procedure definitions
-* **Constants** - EXPRESS constant definitions
+The command extracts `ENTITY` objects from EXPRESS schemas, with the following
+information:
 
-Each extracted term includes:
+. a unique identifier in form of `{schema_name}.{entity_name}` (where `entity_name` is a slug
+  derived from the `ENTITY` name);
+. a generated definition depending on the schema type and entity type;
+. a "Note to entry" from its first coherent textual element described in Annotated EXPRESS;
+. source information including: schema name and schema version;
+. domain in the pattern of: `{domain_type}: {schema_name}`, where `domain_type` is one of
+  `application module`, `resource`, or `business object model`.
 
-* Unique identifier based on schema and term name
-* Term definition and description
-* Source schema information
-* Appropriate domain classification (application module, resource, or business object model)
 
 ==== Output format
 
 The command generates a Glossarist v2 compliant dataset with:
 
-* `concept/` directory containing concept definition files
-* `localized_concept/` directory containing localized concept files
-* YAML format following Glossarist v2 schema specifications
-* Proper cross-references and citations to source schemas
+* `concept/` directory containing concept definition files in YAML
+* `localized_concept/` directory containing localized concept files in YAML
 
 .To extract terms from a schema manifest file
 [example]
@@ -394,7 +398,8 @@ The command generates a Glossarist v2 compliant dataset with:
 [source,sh]
 ----
 $ bundle exec suma extract_terms schemas-smrl-all.yml glossarist_output
-# => generates glossarist_output/concept/*.yaml and glossarist_output/localized_concept/*.yaml
+# => generates glossarist_output/concept/*.yaml and
+#      glossarist_output/localized_concept/*.yaml
 ----
 ====
 
@@ -408,7 +413,6 @@ $ bundle exec suma extract_terms schemas-activity-modules.yml terms_output
 ----
 ====
 
-The generated dataset is meant to be used for ISO 10303-2.
 
 
 == Usage: Ruby

data/lib/suma/cli/extract_terms.rb

@@ -12,6 +12,17 @@ module Suma
     # ExtractTerms command using Expressir to extract terms into the
     # Glossarist v2 format
     class ExtractTerms < Thor
+      # Matches patterns like "A thing is a type of {{entity}}." or
+      # "An object is a type of a {{entity}}"
+      REDUNDANT_NOTE_REGEX =
+        %r{
+          ^An?                   # Starts with "A" or "An"
+          \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
+          (\sa|\san)?            # Optional " a" or " an"
+          \s\{\{[^\}]*\}\}       # Text in double curly braces
+          \s*?\.?$               # Optional whitespace and period at the end
+        }x
+
       desc "extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
            "Extract terms from SCHEMA_MANIFEST_FILE into " \
            "Glossarist v2 format"
@@ -129,11 +140,12 @@ module Suma
           data.sources = [source_ref] if source_ref
 
           # Only assign optional fields if they have content
-          notes = get_entity_notes(entity, schema_domain)
+          notes = get_entity_notes(entity, schema_domain, data.definition)
           data.notes = notes if notes && !notes.empty?
 
-          examples = get_entity_examples(entity, schema_domain)
-          data.examples = examples if examples && !examples.empty?
+          # examples = get_entity_examples(entity, schema_domain)
+          # data.examples = examples if examples && !examples.empty?
+          data.examples = []
         end
 
         Glossarist::LocalizedConcept.new.tap do |concept|
@@ -245,9 +257,20 @@ module Suma
         [Glossarist::DetailedDefinition.new(content: definition)]
       end
 
-      def get_entity_notes(entity, schema_domain)
+      def get_entity_notes(entity, schema_domain, definitions)
+        puts "Extracting notes for entity: #{entity.id}"
         notes = []
 
+        notes = add_entity_notes(entity, schema_domain, notes)
+        # notes = add_other_notes(entity, schema_domain, notes)
+        notes = only_keep_first_sentence(notes)
+        notes = remove_see_content(notes)
+        notes = remove_redundant_note(notes)
+        notes = remove_invalid_references(notes)
+        compare_with_definitions(notes, definitions)
+      end
+
+      def add_entity_notes(entity, schema_domain, notes)
         # Add trimmed definition from entity description as first note
         if entity.remarks && !entity.remarks.empty?
           trimmed_def = trim_definition(entity.remarks)
@@ -258,7 +281,11 @@ module Suma
           end
         end
 
-        # Add other notes
+        notes.compact
+      end
+
+      def add_other_notes(entity, schema_domain, notes)
+        # Add other notes from entity remarks
         other_notes = [
           entity.remark_items&.select do |ri|
             ri.id == "__note"
@@ -274,6 +301,67 @@ module Suma
         notes
       end
 
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 1. First sentence in first paragraph of the entity description
+      # (in EXPRESS remark) becomes NOTE 1 in ISO 10303-2 of the entity.
+      def only_keep_first_sentence(notes)
+        notes.each do |note|
+          # Split by period and take the first sentence
+          # Avoid splitting by pattern like "abc.def"
+          if note&.content
+            new_content = note.content
+              .split(".\n").first.strip
+              .split(". ").first.strip
+            note.content = if new_content.end_with?(".")
+                             new_content
+                           else
+                             "#{new_content}."
+                           end
+          end
+        end
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 2. If this first sentence matches the 7-word magic sentence
+      # (2-3 forms of that), it is discarded so there will not be a NOTE 1.
+      def compare_with_definitions(notes, definitions)
+        if notes&.first&.content == definitions&.first&.content
+          # Discarding first note as it matches the definition
+          return []
+        end
+
+        notes
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 3. No reference to any types or attribute or figures allowed in first
+      # sentence. Entity references “{{…}}” are allowed.
+      def remove_invalid_references(notes)
+        notes.reject do |note|
+          note.content.include?("image::") ||
+            note.content.match?(/<<(.*?){1,999}>>/)
+        end
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 4. Entity notes and examples in EXPRESS remarks are NOT represented in
+      # part 2.
+      def remove_redundant_note(notes)
+        notes.reject do |note|
+          note.content.match?(REDUNDANT_NOTE_REGEX) &&
+            !note.content.include?("\n")
+        end
+      end
+
+      # https://github.com/metanorma/iso-10303/issues/621
+      # 5. If the sentence contains “\s+(see …)”, the contents including the
+      # parentheses are removed.
+      def remove_see_content(notes)
+        notes.each do |note|
+          note.content = note.content.gsub(/\s+\(see(.*?){1,999}\)/, "")
+        end
+      end
+
       def get_entity_examples(entity, schema_domain)
         examples = entity.remark_items&.select do |ri|
           ri.id == "__example"
@@ -382,13 +470,18 @@ module Suma
       end
       # rubocop:enable Metrics/MethodLength
 
-      # Replace `<<express:{schema}.{entity},{render}>>` with {{entity,render}}
+      # Replace `<<express:{schema}.{entity}>>` with {{entity}}
+      # and `<<express:{schema}.{entity},{render}>>` with {{entity,render}}
       def express_reference_to_mention(description)
         # TODO: Use Expressir to check whether the "entity" is really an
         # EXPRESS ENTITY. If not, skip the mention.
-        description.gsub(/<<express:([^,]+),([^>]+)>>/) do |_match|
-          "{{#{Regexp.last_match[1].split('.').last},#{Regexp.last_match[2]}}}"
-        end
+        description
+          .gsub(/<<express:([^,]+)>>/) do |_match|
+            "{{#{Regexp.last_match[1].split('.').last}}}"
+          end.gsub(/<<express:([^,]+),([^>]+)>>/) do |_match|
+            "{{#{Regexp.last_match[1].split('.').last}," \
+              "#{Regexp.last_match[2]}}}"
+          end
       end
 
       def entity_name_to_text(entity_id)

data/lib/suma/cli/generate_schemas.rb

@@ -64,8 +64,6 @@ module Suma
       def load_yaml(file_path)
         YAML.safe_load(
           File.read(file_path, encoding: "UTF-8"),
-          permitted_classes: [Date, Time, Symbol],
-          permitted_symbols: [],
           aliases: true,
         )
       end

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.1.15"
+  VERSION = "0.1.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.17
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-07-07 00:00:00.000000000 Z
+date: 2025-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir