suma 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91d8fb8b814f47412c1374e7c12c9d3fb5368a5275c409355c6cc02a728305b6
-  data.tar.gz: 3cdf5da9b7c251df88f6434222e3a575e6f1cf3839f83ab7cb47091180807510
+  metadata.gz: 2f4f11e324818926bf5846c777a2ae2e1e317f5d174f9fd6f41b0d8d7f955816
+  data.tar.gz: 78fafa5a5247461c7e9907594aaa4cd9b6ce3644966423ec9ded7917f7be2f1c
 SHA512:
-  metadata.gz: 55422c9b98ac260feb77bc8937aff2d45196ba369850a5166b82fec88e7318007c2a6e3d345a441b0ea7e6e3c6df3854432a56ab33012079ababa0fae8a0f6ab
-  data.tar.gz: 50e1425e0bb51cff0866dbb4f7de66c03fc72772660aa253eafa5f25ebcd072e6d7f4c4b5006ad84c78fbd9364eb8cf5494498fc1c6747efe9e2b50a9de8757a
+  metadata.gz: 718b82f083343c86f5dd9e3b7fcf82fab4b918a6911bd29c8b080b24cb6ca7e40e4f13f545ce16728789dd4e7f06d75dca6e1fb702ecde2ed2f3840951b1c356
+  data.tar.gz: 74b9d53b69b1b3008f4eeae03e1811fed2d35fbdbe4108c9f9e368fde96c98d84c8ce3a98b682bdf8669cdd136e4d8c43ad9855537f709f829acfe0f1ed6bcee

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2025-03-23 10:59:23 UTC using RuboCop version 1.74.0.
+# on 2025-04-03 06:48:36 UTC using RuboCop version 1.75.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -13,7 +13,17 @@ Gemspec/DuplicatedAssignment:
   Exclude:
     - 'suma.gemspec'
 
-# Offense count: 43
+# Offense count: 2
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
+# SupportedHashRocketStyles: key, separator, table
+# SupportedColonStyles: key, separator, table
+# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
+Layout/HashAlignment:
+  Exclude:
+    - 'lib/suma/cli/validate.rb'
+
+# Offense count: 55
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https
@@ -21,33 +31,38 @@ Layout/LineLength:
   Exclude:
     - 'lib/suma/cli.rb'
     - 'lib/suma/cli/build.rb'
-    - 'lib/suma/cli/links.rb'
+    - 'lib/suma/cli/validate.rb'
+    - 'lib/suma/cli/validate_ascii.rb'
+    - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/collection_manifest.rb'
     - 'lib/suma/processor.rb'
     - 'lib/suma/schema_attachment.rb'
     - 'lib/suma/schema_collection.rb'
     - 'lib/suma/schema_document.rb'
     - 'lib/suma/thor_ext.rb'
+    - 'spec/suma/cli/validate_ascii_spec.rb'
     - 'suma.gemspec'
 
-# Offense count: 1
-Lint/DuplicateMethods:
+# Offense count: 3
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowInHeredoc.
+Layout/TrailingWhitespace:
   Exclude:
-    - 'lib/suma/express_schema.rb'
+    - 'lib/suma/cli.rb'
+    - 'lib/suma/cli/validate.rb'
 
 # Offense count: 2
-# This cop supports safe autocorrection (--autocorrect).
-# Configuration parameters: AutoCorrect, AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods, NotImplementedExceptions.
-# NotImplementedExceptions: NotImplementedError
-Lint/UnusedMethodArgument:
+Lint/DuplicateMethods:
   Exclude:
-    - 'lib/suma/cli.rb'
+    - 'lib/suma/cli/validate_ascii.rb'
+    - 'lib/suma/express_schema.rb'
 
-# Offense count: 11
+# Offense count: 15
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
 Metrics/AbcSize:
   Exclude:
-    - 'lib/suma/cli/links.rb'
+    - 'lib/suma/cli/validate_ascii.rb'
+    - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/schema_attachment.rb'
     - 'lib/suma/schema_document.rb'
     - 'lib/suma/thor_ext.rb'
@@ -58,28 +73,39 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 64
 
-# Offense count: 2
+# Offense count: 4
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/CyclomaticComplexity:
   Exclude:
-    - 'lib/suma/cli/links.rb'
+    - 'lib/suma/cli/validate_ascii.rb'
+    - 'lib/suma/cli/validate_links.rb'
     - 'lib/suma/thor_ext.rb'
 
-# Offense count: 9
+# Offense count: 21
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
-  Max: 72
+  Max: 107
 
-# Offense count: 3
+# Offense count: 4
 # Configuration parameters: CountKeywordArgs, MaxOptionalParameters.
 Metrics/ParameterLists:
   Max: 6
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
 Metrics/PerceivedComplexity:
   Exclude:
-    - 'lib/suma/cli/links.rb'
+    - 'lib/suma/cli/validate_ascii.rb'
+    - 'lib/suma/cli/validate_links.rb'
+
+# Offense count: 5
+# Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns.
+# SupportedStyles: snake_case, normalcase, non_integer
+# AllowedIdentifiers: TLS1_1, TLS1_2, capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339, x86_64
+Naming/VariableNumber:
+  Exclude:
+    - 'lib/suma/cli/validate_ascii.rb'
+    - 'spec/suma/cli/validate_ascii_spec.rb'
 
 # Offense count: 1
 # Configuration parameters: MinSize.
@@ -87,9 +113,14 @@ Performance/CollectionLiteralInLoop:
   Exclude:
     - 'spec/suma/cli_spec.rb'
 
-# Offense count: 1
+# Offense count: 3
+# Configuration parameters: CountAsOne.
+RSpec/ExampleLength:
+  Max: 16
+
+# Offense count: 5
 RSpec/MultipleExpectations:
-  Max: 2
+  Max: 12
 
 # Offense count: 1
 # This cop supports safe autocorrection (--autocorrect).
@@ -97,4 +128,11 @@ RSpec/MultipleExpectations:
 # SupportedStyles: empty, nil, both
 Style/EmptyElse:
   Exclude:
-    - 'lib/suma/cli/links.rb'
+    - 'lib/suma/cli/validate_links.rb'
+
+# Offense count: 4
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: MaxUnannotatedPlaceholdersAllowed, Mode, AllowedMethods, AllowedPatterns.
+# SupportedStyles: annotated, template, unannotated
+Style/FormatStringToken:
+  EnforcedStyle: unannotated

data/README.adoc

@@ -37,9 +37,10 @@ $ gem install suma
 # Defaults to `suma help`
 $ suma
 Commands:
-  suma build METANORMA_SITE_MANIFEST  # Build collection specified in site manifest (`metanorma*.yml`)
-  suma links SUBCOMMAND ...ARGS       # Manage EXPRESS links
-  suma help [COMMAND]                 # Describe available commands or one specific command
+  suma build METANORMA_SITE_MANIFEST      # Build collection specified in site manifest (`metanorma*.yml`)
+  suma reformat EXPRESS_FILE_PATH         # Reformat EXPRESS files
+  suma validate SUBCOMMAND ...ARGS        # Validate express documents
+  suma help [COMMAND]                     # Describe available commands or one specific command
 ----
 
 === Build command
@@ -89,19 +90,61 @@ $ bundle exec suma build --no-compile metanorma-srl.yml
 All documents need to have a `schemas.yaml` in their document root that lists
 out which schemas the document includes.
 
-=== Links command
+=== Reformat command
+
+The `reformat` command provides utilities for reformatting EXPRESS files.
+
+[source,sh]
+----
+$ suma reformat EXPRESS_FILE_PATH [options]
+----
+
+Parameters:
+
+`EXPRESS_FILE_PATH`:: Path to an EXPRESS file or a folder containing EXPRESS
+files
+
+Options:
+
+`--[no-]recursive`, `-r`:: Select EXPRESS files recursively based on the specified
+folder path (default: false)
+
+[example]
+====
+.To reformat all EXPRESS files under the current directory recursively
+[source,sh]
+----
+$ bundle exec suma reformat `pwd` -r
+----
+====
+
+This command:
+
+* Loads the EXPRESS files specified in the `EXPRESS_FILE_PATH`
+* Reformats and saves the loaded EXPRESS files
+
+=== Validate command
 
 ==== General
 
-The `links` command provides utilities for managing EXPRESS links.
+The `validate` command groups various validation utilities for EXPRESS documents.
+
+[source,sh]
+----
+$ suma validate SUBCOMMAND [options]
+----
+
+Subcommands:
+- `links` - Validate EXPRESS links
+- `ascii` - Check for non-ASCII characters in EXPRESS files
 
-==== Extract and validate
+==== Links subcommand
 
-Extracts and validates EXPRESS links without creating intermediate files.
+The `links` subcommand extracts and validates EXPRESS links without creating intermediate files.
 
 [source,sh]
 ----
-$ suma links extract_and_validate SCHEMAS_FILE DOCUMENTS_PATH [OUTPUT_FILE]
+$ suma validate links SCHEMAS_FILE DOCUMENTS_PATH [OUTPUT_FILE]
 ----
 
 Parameters:
@@ -117,7 +160,7 @@ Parameters:
 .To validate EXPRESS links in documents
 [source,sh]
 ----
-$ bundle exec suma links extract_and_validate schemas-srl.yml documents validation_results.txt
+$ bundle exec suma validate links schemas-srl.yml documents validation_results.txt
 ----
 ====
 
@@ -129,16 +172,13 @@ This command:
 * Writes validation results to the `OUTPUT_FILE`
 * Provides progress bars to track schema loading and link validation
 
+==== ASCII subcommand
 
-=== Reformat command
-
-==== General
-
-The `reformat` command provides utilities for reformatting EXPRESS files.
+The `ascii` subcommand detects non-ASCII characters in EXPRESS files and reports on those exact lines, providing replacement suggestions.
 
 [source,sh]
 ----
-$ suma reformat EXPRESS_FILE_PATH [options]
+$ suma validate ascii EXPRESS_FILE_PATH [options]
 ----
 
 Parameters:
@@ -148,22 +188,103 @@ files
 
 Options:
 
-`--[no-]recursive`:: Select EXPRESS files recursively based on the specified
+`--[no-]recursive`, `-r`:: Select EXPRESS files recursively based on the specified
 folder path (default: false)
+`--[no-]yaml`, `-y`:: Output results in YAML format for machine processing (default: false)
 
 [example]
 ====
-.To reformat all EXPRESS files under the current directory recursively
+.To validate all EXPRESS files in a specific directory recursively
 [source,sh]
 ----
-$ bundle exec suma reformat `pwd` -r
+$ bundle exec suma validate ascii ../iso-10303/schemas -r
+----
+
+.To validate and output results in YAML format
+[source,sh]
+----
+$ bundle exec suma validate ascii ../iso-10303/schemas -r -y > validation.yml
 ----
 ====
 
 This command:
 
 * Loads the EXPRESS files specified in the `EXPRESS_FILE_PATH`
-* Reformats and saves the loaded EXPRESS files
+* Scans each line for non-ASCII characters
+* Reports detailed information about each violation, including:
+** Filename and line number
+** The exact line content
+** Visual indication of the non-ASCII sequence location
+** Character details with hexadecimal representation
+* Provides specific replacement suggestions:
+** For math symbols: provides equivalent AsciiMath notation
+** For other non-ASCII characters: provides ISO 10303-11 encoded string literal format
+* Displays a summary table showing:
+** File path (directory/filename)
+** Each non-ASCII symbol found
+** Suggested replacement for each symbol
+** Number of occurrences of each character
+** Totals row showing unique character count and overall occurrences
+* Summarizes findings across all scanned files
+* Optionally outputs structured data in YAML format with detailed occurrence information
+
+Human-readable output format example:
+
+[source,text]
+----
+/path/to/file.exp:
+  Line 42, Column 15:
+    ENTITY some_entity (name: STRING, description: "résumé");
+                                       ^^^^^
+      "é" - Hex: 0xe9, UTF-8 bytes: 0xc3 0xa9
+      Replacement: ISO 10303-11: "000000E9"
+
+      "s" - Hex: 0x73, UTF-8 bytes: 0x73
+
+      "u" - Hex: 0x75, UTF-8 bytes: 0x75
+
+      "m" - Hex: 0x6d, UTF-8 bytes: 0x6d
+
+      "é" - Hex: 0xe9, UTF-8 bytes: 0xc3 0xa9
+      Replacement: ISO 10303-11: "000000E9"
+
+  Found 1 non-ASCII sequence(s) in file.exp
+
+Summary:
+  Scanned 3 EXPRESS file(s)
+  Found 1 non-ASCII sequence(s) in 1 file(s)
+
++------------------+--------------------+-----------------------------+-------------+
+|       File       |       Symbol       |        Replacement          | Occurrences |
++------------------+--------------------+-----------------------------+-------------+
+| path/to/file.exp | "é" (0xe9)         | ISO 10303-11: "000000E9"    | 2           |
++------------------+--------------------+-----------------------------+-------------+
+| TOTAL            | 1 unique           |                             | 2           |
++------------------+--------------------+-----------------------------+-------------+
+----
+
+===== Japanese Character Example
+
+For Japanese characters like 神戸 (Kobe), the command will provide ISO 10303-11 encoded string literal replacements:
+
+[source,text]
+----
+"神" - Hex: 0x795e, UTF-8 bytes: 0xe7 0xa5 0x9e
+Replacement: ISO 10303-11: "0000795E"
+
+"戸" - Hex: 0x6238, UTF-8 bytes: 0xe6 0x88 0xb8
+Replacement: ISO 10303-11: "00006238"
+----
+
+===== Math Symbol Example
+
+For mathematical symbols, the command will provide equivalent AsciiMath notation:
+
+[source,text]
+----
+"×" - Hex: 0xd7, UTF-8 bytes: 0xc3 0x97
+Replacement: AsciiMath: xx
+----
 
 
 == Usage: Ruby

data/lib/suma/cli/validate.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module Suma
+  module Cli
+    # Main validate command that groups the validation subcommands
+    class Validate < Thor
+      desc "links SCHEMAS_FILE DOCUMENTS_PATH [OUTPUT_FILE]",
+           "Extract and validate express links without creating intermediate file"
+      def links(*args)
+        require_relative "validate_links"
+
+        # Forward the command to ValidateLinks
+        links = Cli::ValidateLinks.new
+        links.extract_and_validate(*args)
+      end
+
+      desc "ascii EXPRESS_FILE_PATH",
+           "Validate EXPRESS files for ASCII-only content"
+      option :recursive, type: :boolean, default: false, aliases: "-r",
+                         desc: "Validate EXPRESS files under the specified path recursively"
+      option :yaml, type: :boolean, default: false, aliases: "-y",
+                    desc: "Output results in YAML format"
+      def ascii(express_file_path)
+        require_relative "validate_ascii"
+
+        validator = Cli::ValidateAscii.new
+        validator.options = options
+        validator.validate_ascii(express_file_path)
+      end
+    end
+  end
+end

data/lib/suma/cli/validate_ascii.rb

@@ -0,0 +1,519 @@
+# frozen_string_literal: true
+
+require "thor"
+require "yaml"
+require "terminal-table"
+require "plurimath"
+require "set" # For using Set in unique_character_count
+require_relative "../thor_ext"
+
+module Suma
+  module Cli
+    # Represents a non-ASCII character with its details and replacement
+    class NonAsciiCharacter
+      attr_reader :char, :hex, :utf8, :is_math, :replacement,
+                  :replacement_type, :occurrences
+
+      def initialize(char, hex, utf8, is_math, replacement, replacement_type)
+        @char = char
+        @hex = hex
+        @utf8 = utf8
+        @is_math = is_math
+        @replacement = replacement
+        @replacement_type = replacement_type
+        @occurrences = []
+      end
+
+      def add_occurrence(line_number, column, line)
+        @occurrences << {
+          line_number: line_number,
+          column: column,
+          line: line,
+        }
+      end
+
+      def replacement_text
+        @is_math ? "AsciiMath: #{@replacement}" : "ISO 10303-11: #{@replacement}"
+      end
+
+      def occurrence_count
+        @occurrences.size
+      end
+
+      def to_h
+        {
+          character: @char,
+          hex: @hex,
+          utf8: @utf8,
+          is_math: @is_math,
+          replacement_type: @replacement_type,
+          replacement: @replacement,
+          occurrence_count: occurrence_count,
+          occurrences: @occurrences,
+        }
+      end
+    end
+
+    # Represents all non-ASCII characters in a file
+    class FileViolations
+      attr_reader :path, :filename, :directory, :violations, :unique_characters
+
+      def initialize(file_path)
+        @path = file_path
+        @filename = File.basename(file_path)
+        @directory = File.dirname(file_path)
+        @characters = {}  # Map of characters to NonAsciiCharacter objects
+        @violations = []  # List of violations (line, column, etc.)
+      end
+
+      def add_violation(line_number, column, match, char_details, line)
+        violation = {
+          line_number: line_number,
+          column: column,
+          match: match,
+          char_details: char_details,
+          line: line,
+        }
+
+        @violations << violation
+
+        # Register each character
+        char_details.each do |detail|
+          char = detail[:char]
+          unless @characters[char]
+            @characters[char] = NonAsciiCharacter.new(
+              char,
+              detail[:hex],
+              detail[:utf8],
+              detail[:is_math],
+              detail[:replacement],
+              detail[:replacement_type],
+            )
+          end
+
+          @characters[char].add_occurrence(line_number, column, line)
+        end
+      end
+
+      def violation_count
+        @violations.size
+      end
+
+      def unique_characters
+        @characters.values
+      end
+
+      def display_path
+        "#{File.basename(@directory)}/#{@filename}"
+      end
+
+      def full_path
+        File.expand_path(@path)
+      end
+
+      def to_h
+        {
+          file: display_path,
+          count: violation_count,
+          non_ascii_characters: unique_characters.map(&:to_h),
+        }
+      end
+    end
+
+    # Collection of all violations across multiple files
+    class NonAsciiViolationCollection
+      attr_reader :file_violations, :total_files
+
+      def initialize
+        @file_violations = {} # Map of file paths to FileViolations objects
+        @total_files = 0
+        @unicode_to_asciimath = nil
+      end
+
+      def process_file(file)
+        @total_files += 1
+
+        # Initialize the mapping once
+        @unicode_to_asciimath ||= build_unicode_to_asciimath_map
+
+        file_violations = process_file_violations(file)
+        return if file_violations.violations.empty?
+
+        @file_violations[file] = file_violations
+      end
+
+      def files_with_violations
+        @file_violations.size
+      end
+
+      def total_violations
+        @file_violations.values.sum(&:violation_count)
+      end
+
+      def unique_character_count
+        # Get total unique characters across all files
+        all_chars = Set.new
+        @file_violations.each_value do |file_violation|
+          file_violation.unique_characters.each do |char|
+            all_chars.add(char.char)
+          end
+        end
+        all_chars.size
+      end
+
+      def total_occurrence_count
+        # Sum all occurrences of all characters across all files
+        @file_violations.values.sum do |file_violation|
+          file_violation.unique_characters.sum(&:occurrence_count)
+        end
+      end
+
+      def to_yaml_data
+        {
+          summary: {
+            total_files: @total_files,
+            files_with_violations: files_with_violations,
+            total_violations: total_violations,
+            total_unique_characters: unique_character_count,
+            total_occurrences: total_occurrence_count,
+          },
+          violations: @file_violations.transform_keys do |k|
+            File.expand_path(k)
+          end.transform_values(&:to_h),
+        }
+      end
+
+      def print_text_output
+        return if @file_violations.empty?
+
+        # Print each file's violations
+        @file_violations.each_value do |file_violation|
+          puts "\n#{file_violation.display_path}:"
+
+          file_violation.violations.each do |v|
+            puts "  Line #{v[:line_number]}, Column #{v[:column]}:"
+            puts "    #{v[:line]}"
+            puts "    #{' ' * v[:column]}#{'^' * v[:match].length} Non-ASCII sequence"
+
+            v[:char_details].each do |cd|
+              character = file_violation.unique_characters.find do |c|
+                c.char == cd[:char]
+              end
+              next unless character
+
+              puts "      \"#{cd[:char]}\" - Hex: #{cd[:hex]}, UTF-8 bytes: #{cd[:utf8]}"
+              puts "      Replacement: #{character.replacement_text}"
+            end
+            puts ""
+          end
+
+          puts "  Found #{file_violation.violation_count} non-ASCII sequence(s) in #{file_violation.filename}\n"
+        end
+
+        # Print summary
+        puts "\nSummary:"
+        puts "  Scanned #{@total_files} EXPRESS file(s)"
+        puts "  Found #{total_violations} non-ASCII sequence(s) in #{files_with_violations} file(s)"
+      end
+
+      def print_table_output
+        return if @file_violations.empty?
+
+        table = ::Terminal::Table.new(
+          title: "Non-ASCII Characters Summary",
+          headings: ["File", "Symbol", "Replacement", "Occurrences"],
+        )
+
+        total_occurrences = 0
+
+        @file_violations.each_value do |file_violation|
+          file_violation.unique_characters.each do |character|
+            occurrence_count = character.occurrence_count
+            total_occurrences += occurrence_count
+
+            table.add_row [
+              file_violation.display_path,
+              "\"#{character.char}\" (#{character.hex})",
+              character.replacement_text,
+              occurrence_count,
+            ]
+          end
+        end
+
+        # Add a separator and total row
+        table.add_separator
+        table.add_row [
+          "TOTAL",
+          "#{unique_character_count} unique",
+          "",
+          total_occurrences,
+        ]
+
+        puts "\n#{table}\n"
+      end
+
+      private
+
+      def process_file_violations(file)
+        file_violations = FileViolations.new(file)
+
+        # Process file line by line
+        File.readlines(file,
+                       encoding: "UTF-8").each_with_index do |line, line_idx|
+          line_number = line_idx + 1
+
+          # Skip if line only contains ASCII
+          next unless /[^\x00-\x7F]/.match?(line)
+
+          # Find all non-ASCII sequences
+          line.chomp.scan(/([^\x00-\x7F]+)/) do |match|
+            match = match[0]
+            column = line.index(match)
+
+            # Process each character in the sequence
+            char_details = match.chars.filter_map do |c|
+              process_non_ascii_char(c)
+            end
+
+            # Skip if no non-ASCII characters found
+            next if char_details.empty?
+
+            file_violations.add_violation(line_number, column, match,
+                                          char_details, line.chomp)
+          end
+        end
+
+        file_violations
+      end
+
+      def process_non_ascii_char(char)
+        # Skip ASCII characters
+        return nil if char.ord <= 0x7F
+
+        code_point = char.ord
+        hex = "0x#{code_point.to_s(16)}"
+        utf8 = code_point.chr(Encoding::UTF_8).bytes.map do |b|
+          "0x#{b.to_s(16)}"
+        end.join(" ")
+
+        # Check if it's a math symbol
+        if asciimath = @unicode_to_asciimath[char]
+          return {
+            char: char,
+            hex: hex,
+            utf8: utf8,
+            is_math: true,
+            replacement: asciimath,
+            replacement_type: "asciimath",
+          }
+        end
+
+        # Not a math symbol, use ISO encoding
+        {
+          char: char,
+          hex: hex,
+          utf8: utf8,
+          is_math: false,
+          replacement: encode_iso_10303_11(char),
+          replacement_type: "iso-10303-11",
+        }
+      end
+
+      def encode_iso_10303_11(char)
+        code_point = char.ord
+
+        # Format the encoded value with double quotes
+        if code_point < 0x10000
+          "\"#{sprintf('%08X', code_point)}\"" # e.g., "00000041" for 'A'
+        else
+          # For higher code points, use all four octets
+          group = (code_point >> 24) & 0xFF
+          plane = (code_point >> 16) & 0xFF
+          row = (code_point >> 8) & 0xFF
+          cell = code_point & 0xFF
+
+          "\"#{sprintf('%02X%02X%02X%02X', group, plane, row, cell)}\""
+        end
+      end
+
+      def build_unicode_to_asciimath_map
+        # Start with a pre-defined mapping of common math symbols
+        unicode_to_asciimath = {
+          # Greek letters
+          "α" => "alpha",
+          "β" => "beta",
+          "γ" => "gamma",
+          "Γ" => "Gamma",
+          "δ" => "delta",
+          "Δ" => "Delta",
+          "ε" => "epsilon",
+          "ζ" => "zeta",
+          "η" => "eta",
+          "θ" => "theta",
+          "Θ" => "Theta",
+          "ι" => "iota",
+          "κ" => "kappa",
+          "λ" => "lambda",
+          "Λ" => "Lambda",
+          "μ" => "mu",
+          "ν" => "nu",
+          "ξ" => "xi",
+          "Ξ" => "Xi",
+          "π" => "pi",
+          "Π" => "Pi",
+          "ρ" => "rho",
+          "σ" => "sigma",
+          "Σ" => "Sigma",
+          "τ" => "tau",
+          "υ" => "upsilon",
+          "φ" => "phi",
+          "Φ" => "Phi",
+          "χ" => "chi",
+          "ψ" => "psi",
+          "Ψ" => "Psi",
+          "ω" => "omega",
+          "Ω" => "Omega",
+
+          # Math operators
+          "×" => "xx",
+          "÷" => "div",
+          "±" => "pm",
+          "∓" => "mp",
+          "∞" => "oo",
+          "≤" => "le",
+          "≥" => "ge",
+          "≠" => "ne",
+          "≈" => "~~",
+          "≅" => "cong",
+          "≡" => "equiv",
+          "∈" => "in",
+          "∉" => "notin",
+          "⊂" => "subset",
+          "⊃" => "supset",
+          "∩" => "cap",
+          "∪" => "cup",
+          "∧" => "and",
+          "∨" => "or",
+          "¬" => "neg",
+          "∀" => "forall",
+          "∃" => "exists",
+          "∄" => "nexists",
+          "∇" => "grad",
+          "∂" => "del",
+          "∑" => "sum",
+          "∏" => "prod",
+          "∫" => "int",
+          "∮" => "oint",
+          "√" => "sqrt",
+          "⊥" => "perp",
+          "‖" => "norm",
+          "→" => "rarr",
+          "←" => "larr",
+          "↔" => "harr",
+          "⇒" => "rArr",
+          "⇐" => "lArr",
+          "⇔" => "hArr",
+        }
+
+        # Augment with symbols from Plurimath
+        begin
+          # Get all symbols supported by AsciiMath
+          Plurimath::Utility.symbols_files.each do |symbol_class|
+            symbol = symbol_class.new
+
+            # Get the Unicode and AsciiMath representations
+            unicodes = symbol.to_unicodemath
+            asciimaths = symbol.to_asciimath
+
+            # Skip if either representation is missing
+            next unless unicodes.is_a?(Array) && asciimaths.is_a?(Array)
+            # Skip if empty arrays
+            next if unicodes.empty? || asciimaths.empty?
+
+            unicodes.each_with_index do |unicode, index|
+              # Skip if we're beyond available AsciiMath representations
+              next if index >= asciimaths.length
+              # Skip empty string values
+              next if unicode.to_s.empty?
+
+              # Map each character to its AsciiMath equivalent
+              unicode.to_s.chars.each do |char|
+                # Only add if not already in our mapping
+                unicode_to_asciimath[char] ||= asciimaths[index]
+              end
+            end
+          rescue StandardError => e
+            # Skip this symbol class if there's an error
+            puts "Warning: Error processing symbol class #{symbol_class}: #{e.message}" if $DEBUG
+          end
+        rescue StandardError => e
+          # Continue even if Plurimath integration fails
+          puts "Warning: Error loading Plurimath symbols: #{e.message}" if $DEBUG
+        end
+
+        unicode_to_asciimath
+      end
+    end
+
+    # ValidateAscii command for checking EXPRESS files for non-ASCII characters
+    class ValidateAscii < Thor
+      desc "validate-ascii EXPRESS_FILE_PATH",
+           "Validate EXPRESS files for ASCII-only content"
+      option :recursive, type: :boolean, default: false, aliases: "-r",
+                         desc: "Validate EXPRESS files under the specified " \
+                               "path recursively"
+      option :yaml, type: :boolean, default: false, aliases: "-y",
+                    desc: "Output results in YAML format"
+
+      def validate_ascii(express_file_path) # rubocop:disable Metrics/AbcSize
+        if File.file?(express_file_path)
+          unless File.exist?(express_file_path)
+            raise Errno::ENOENT, "Specified EXPRESS file " \
+                                 "`#{express_file_path}` not found."
+          end
+
+          if File.extname(express_file_path) != ".exp"
+            raise ArgumentError, "Specified file `#{express_file_path}` is " \
+                                 "not an EXPRESS file."
+          end
+
+          exp_files = [express_file_path]
+        elsif options[:recursive]
+          # Support the relative path with glob pattern
+          base_path = File.expand_path(express_file_path)
+          exp_files = Dir.glob("#{base_path}/**/*.exp")
+        else
+          # Non-recursive option
+          base_path = File.expand_path(express_file_path)
+          exp_files = Dir.glob("#{base_path}/*.exp")
+        end
+
+        if exp_files.empty?
+          raise Errno::ENOENT, "No EXPRESS files found in " \
+                               "`#{express_file_path}`."
+        end
+
+        run(exp_files)
+      end
+
+      private
+
+      def run(exp_files)
+        # Process all files and collect violations
+        collection = NonAsciiViolationCollection.new
+
+        exp_files.each do |exp_file|
+          collection.process_file(exp_file)
+        end
+
+        # Output results based on format
+        if options[:yaml]
+          puts collection.to_yaml_data.to_yaml
+        else
+          collection.print_text_output
+          collection.print_table_output if collection.files_with_violations.positive?
+        end
+      end
+    end
+  end
+end

data/lib/suma/cli/{links.rb → validate_links.rb}

@@ -5,8 +5,8 @@ require_relative "../utils"
 
 module Suma
   module Cli
-    # Links command for managing EXPRESS links
-    class Links < Thor
+    # ValidateLinks command for managing EXPRESS links
+    class ValidateLinks < Thor
       desc "extract_and_validate SCHEMAS_FILE DOCUMENTS_PATH [OUTPUT_FILE]",
            "Extract and validate express links without creating intermediate file"
       def extract_and_validate(schemas_file = "schemas-srl.yml",

data/lib/suma/cli.rb

@@ -2,6 +2,7 @@
 
 require "thor"
 require_relative "thor_ext"
+require_relative "cli/validate"
 
 module Suma
   module Cli
@@ -22,12 +23,6 @@ module Suma
         Cli::Build.start
       end
 
-      desc "links SUBCOMMAND ...ARGS", "Manage EXPRESS links"
-      def links(*_args)
-        require_relative "cli/links"
-        Cli::Links.start
-      end
-
       desc "reformat EXPRESS_FILE_PATH",
            "Reformat EXPRESS files"
       option :recursive, type: :boolean, default: false, aliases: "-r",
@@ -37,6 +32,13 @@ module Suma
         require_relative "cli/reformat"
         Cli::Reformat.start
       end
+
+      desc "validate SUBCOMMAND ...ARGS", "Validate express documents"
+      subcommand "validate", Cli::Validate
+
+      def self.exit_on_failure?
+        true
+      end
     end
   end
 end

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 end

data/suma.gemspec

@@ -36,7 +36,9 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
   spec.add_dependency "expressir", "~> 2.1"
   spec.add_dependency "lutaml-model", "~> 0.7"
   spec.add_dependency "metanorma-cli"
+  spec.add_dependency "plurimath"
   spec.add_dependency "ruby-progressbar"
+  spec.add_dependency "terminal-table", "~> 3.0"
   spec.add_dependency "thor", ">= 0.20"
   spec.metadata["rubygems_mfa_required"] = "true"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-27 00:00:00.000000000 Z
+date: 2025-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: plurimath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -106,8 +134,10 @@ files:
 - lib/suma.rb
 - lib/suma/cli.rb
 - lib/suma/cli/build.rb
-- lib/suma/cli/links.rb
 - lib/suma/cli/reformat.rb
+- lib/suma/cli/validate.rb
+- lib/suma/cli/validate_ascii.rb
+- lib/suma/cli/validate_links.rb
 - lib/suma/collection_config.rb
 - lib/suma/collection_manifest.rb
 - lib/suma/express_schema.rb