suma 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fcc000e30cb69dd5ab1d326cbcf06ca4fe2c2855a7368f2a0387a369c15721c
-  data.tar.gz: 3873c8355f19b4ccb2f8b8d3530eeac2ad81e65e880e44b97765413377ff71a1
+  metadata.gz: d52aaad6ea81bb43e193cdf0c8e190dfe02f71791e218314a401ae3bea254cd3
+  data.tar.gz: a573936a7eabfad21a4768e6c32869ecc286a2980feeaf8350720b674c604f34
 SHA512:
-  metadata.gz: 0a18e25ee8e47d39028d0bd9bd5dd9c2f602bf540f77b3729f402bf22c86cedc5f7b037c5901886c109d869aa64d1eb82ca70c4960ba73cc2e655bd73867f80a
-  data.tar.gz: 0b29d4b9a641de7a7056785057379e9c26a4709cec7597aa2ee2135f22cc14b52dfd66c78b5ad3204848d775b7e3b3e0537ca34c21defb29155e8dd2a20d8342
+  metadata.gz: d7278d60481858a6507bf97f738df21f0168c60c6453f9de4479545c589450f63c148923be20b31a8bb88e50344adbf8521cfe12a826dbff3103e492782727f3
+  data.tar.gz: b37f9d1f0676607011f6f2059f0b639a563273930eb7961ee52866c04b30df14a7b88076752f4bf2ff4cfd4911a402b547b1c486c42cd9cc381d4e631bc6aac4

data/README.adoc

@@ -37,10 +37,12 @@ $ gem install suma
 # Defaults to `suma help`
 $ suma
 Commands:
-  suma build METANORMA_SITE_MANIFEST      # Build collection specified in site manifest (`metanorma*.yml`)
-  suma reformat EXPRESS_FILE_PATH         # Reformat EXPRESS files
-  suma validate SUBCOMMAND ...ARGS        # Validate express documents
-  suma help [COMMAND]                     # Describe available commands or one specific command
+  suma build METANORMA_SITE_MANIFEST        # Build collection specified in site manifest (`metanorma*.yml`)
+  suma reformat EXPRESS_FILE_PATH           # Reformat EXPRESS files
+  suma validate SUBCOMMAND ...ARGS          # Validate express documents
+  suma generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE # Generate schemas manifest file from Metanorma manifest YAML file
+  suma extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH # Extract terms from schema manifest file
+  suma help [COMMAND]                       # Describe available commands or one specific command
 ----
 
 === Build command
@@ -287,6 +289,85 @@ Replacement: AsciiMath: xx
 ----
 
 
+=== Generate schemas command
+
+The `suma generate_schemas` command generates a schema manifest file containing
+all schemas defined in the Metanorma manifest file.
+
+[source,sh]
+----
+$ suma generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE [options]
+----
+
+Parameters:
+
+`METANORMA_MANIFEST_FILE`:: Path to the Metanorma manifest file
+(e.g.: "metanorma-smrl-all.yml")
+
+Options:
+
+`--exclude_path`, `-e`:: Exclude schemas by pattern (e.g. `*_lf.exp`)
+
+[example]
+====
+.To generate schemas manifest file from Metanorma manifest file
+[source,sh]
+----
+$ bundle exec suma generate_schemas metanorma-smrl-all.yml schemas-smrl-all.yml
+# => generates schemas-smrl-all.yml
+----
+
+.To generate schemas manifest file from Metanorma manifest file and exclude schemas with names like `*_lf.exp`
+[source,sh]
+----
+$ bundle exec suma generate_schemas metanorma-smrl-all.yml schemas-smrl-all.yml -e *_lf.exp
+# => generates schemas-smrl-all.yml without schemas with names like *_lf.exp
+----
+====
+
+All documents need to have a `schemas.yaml` in their document root that lists
+out which schemas the document includes.
+
+
+=== Extract terms command
+
+The `suma extract_terms` command extracts terms from SCHEMA_MANIFEST_FILE and
+generates Glossarist v2 dataset in the output directory.
+
+[source,sh]
+----
+$ suma extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH [options]
+----
+
+Parameters:
+
+`SCHEMA_MANIFEST_FILE`:: Path to SCHEMA_MANIFEST_FILE
+
+`GLOSSARIST_OUTPUT_PATH`:: Path to the output directory for the Glossarist v2
+dataset
+
+Options:
+
+`language_code`:: Language code for the Glossarist (default: "eng")
+
+[example]
+====
+.To extract terms from SCHEMA_MANIFEST_FILE
+[source,sh]
+----
+$ bundle exec suma extract_terms path/to/schemas-smrl-all.yml glossarist_output
+# => generates glossarist_output/concept/foo.yaml and glossarist_output/localized_concept/bar.yaml
+----
+
+.To extract terms from SCHEMA_MANIFEST_FILE with language code "fra"
+[source,sh]
+----
+$ bundle exec suma extract_terms path/to/schemas-smrl-all.yml glossarist_output -l fra
+# => generates glossarist_output/concept/foo.yaml and glossarist_output/localized_concept/bar.yaml
+----
+====
+
+
 == Usage: Ruby
 
 === General

data/lib/suma/cli/extract_terms.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require "thor"
+require_relative "../thor_ext"
+require "fileutils"
+require "expressir"
+require "yaml"
+require "securerandom"
+require "glossarist"
+
+module Suma
+  module Cli
+    # ExtractTerms command using Expressir to extract terms into the
+    # Glossarist v2 format
+    class ExtractTerms < Thor
+      desc "extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
+           "Extract terms from SCHEMA_MANIFEST_FILE into " \
+           "Glossarist v2 format"
+      option :language_code, type: :string, default: "eng", aliases: "-l",
+                             desc: "Language code for the Glossarist"
+
+      YAML_FILE_EXTENSIONS = [".yaml", ".yml"].freeze
+
+      def extract_terms(schema_manifest_file, output_path) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        language_code = options[:language_code]
+        schema_manifest_file = File.expand_path(schema_manifest_file)
+
+        if File.file?(schema_manifest_file)
+          unless File.exist?(schema_manifest_file)
+            raise Errno::ENOENT, "Specified SCHEMA_MANIFEST_FILE " \
+                                 "`#{schema_manifest_file}` not found."
+          end
+
+          if !YAML_FILE_EXTENSIONS.include?(File.extname(schema_manifest_file))
+            raise ArgumentError, "Specified SCHEMA_MANIFEST_FILE " \
+                                 "`#{schema_manifest_file}` " \
+                                 "is not a YAML file."
+          end
+        end
+
+        run(schema_manifest_file, output_path, language_code)
+      end
+
+      private
+
+      def run(schema_manifest_file, output_path, language_code = "eng")
+        exp_files = get_exp_files(schema_manifest_file)
+
+        exp_files.map do |exp_file|
+          extract(exp_file, output_path, language_code)
+        end
+      end
+
+      def get_exp_files(schema_manifest_file)
+        data = YAML.safe_load(
+          File.read(schema_manifest_file, encoding: "UTF-8"),
+          permitted_classes: [Date, Time, Symbol],
+          permitted_symbols: [],
+          aliases: true,
+        )
+
+        paths = data["schemas"].values.filter_map { |v| v["path"] }
+
+        if paths.empty?
+          raise Errno::ENOENT, "No EXPRESS files found in " \
+                               "`#{schema_manifest_file}`."
+        end
+
+        # resolve paths relative to the directory of the schema manifest file
+        paths.map do |path|
+          File.expand_path(path, File.dirname(schema_manifest_file))
+        end
+      end
+
+      def extract(exp_file, output_path, language_code) # rubocop:disable Metrics/AbcSize
+        puts "Processing EXPRESS file: #{exp_file}"
+        repo = Expressir::Express::Parser.from_file(exp_file)
+        schema = get_default_schema(repo)
+
+        collection = build_managed_concept_collection(
+          schema, language_code
+        )
+
+        output_data(collection, output_path, exp_file)
+      end
+
+      def output_data(collection, output_path, exp_file)
+        unless File.exist?(output_path)
+          FileUtils.mkdir_p(File.expand_path(output_path))
+        end
+
+        puts "Saving collection to files in: #{File.expand_path(output_path)}"
+        collection.save_to_files(File.expand_path(output_path))
+
+        puts "Processing EXPRESS file: #{exp_file}...Done."
+        collection
+      end
+
+      def build_managed_concept_collection(schema, language_code) # rubocop:disable Metrics/AbcSize
+        managed_concept_data = Glossarist::ManagedConceptData.new
+        managed_concept_data.id = get_identifier(schema)
+
+        localized_concept_id = SecureRandom.uuid
+        localized_concept = build_localized_concept(
+          schema, language_code, localized_concept_id
+        )
+
+        managed_concept_data
+          .localizations[localized_concept.language_code] = localized_concept
+
+        managed_concept_data.localized_concepts = {
+          localized_concept.language_code => localized_concept_id,
+        }
+
+        managed_concept = Glossarist::ManagedConcept.new
+        managed_concept.uuid = SecureRandom.uuid
+        managed_concept.data = managed_concept_data
+
+        collection = Glossarist::ManagedConceptCollection.new
+        collection.store(managed_concept)
+        collection
+      end
+
+      def build_localized_concept(schema, language_code, localized_concept_id) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
+        schema_domain = get_domain(schema)
+
+        localized_concept_data = Glossarist::ConceptData.new
+        localized_concept_data.terms = get_terms(schema) || []
+        localized_concept_data.definition = get_definitions(schema) || []
+        localized_concept_data.notes = get_notes(schema, schema_domain) || []
+        localized_concept_data.examples = get_examples(schema,
+                                                       schema_domain) || []
+        localized_concept_data.language_code = language_code
+        localized_concept_data.domain = schema_domain
+        localized_concept_data.sources = get_source_ref(schema) || []
+
+        localized_concept = Glossarist::LocalizedConcept.new
+        localized_concept.data = localized_concept_data
+
+        localized_concept.uuid = localized_concept_id
+        localized_concept
+      end
+
+      def get_default_schema(repo)
+        repo.schemas.first
+      end
+
+      def get_identifier(schema)
+        remark_item = schema.remark_items.find do |s|
+          s.id == "__identifier"
+        end
+        remark_item.remarks.first || SecureRandom.uuid
+      end
+
+      def get_title(schema)
+        remark_item = schema.remark_items.find do |s|
+          s.id == "__title"
+        end
+        remark_item.remarks.first
+      end
+
+      def get_source_ref(schema)
+        remark_item = schema.remark_items.find do |s|
+          s.id == "__published_in"
+        end
+        ref = remark_item&.remarks&.first
+
+        if ref
+          Glossarist::ConceptSource.new(
+            type: "authoritative",
+            origin: Glossarist::Citation.new(ref: ref),
+          )
+        end
+      end
+
+      def get_domain(schema)
+        prefix = module?(schema) ? "application module" : "resource"
+        "#{prefix}: #{schema.id}"
+      end
+
+      def module?(schema)
+        remark_item = schema.remark_items.find do |s|
+          s.id == "__schema_file"
+        end
+
+        File.basename(remark_item.remarks.first, ".*") == "module"
+      end
+
+      def arm?(schema_id)
+        schema_id.end_with?("_arm")
+      end
+
+      def get_terms(schema)
+        schema_title = get_title(schema)
+        if schema_title
+          [
+            Glossarist::Designation::Base.new(
+              designation: schema_title,
+              type: "expression",
+              normative_status: "preferred",
+            ),
+          ]
+        end
+      end
+
+      def get_schema_type(schema)
+        return "resource" if !module?(schema)
+
+        return "arm" if arm?(schema.id)
+
+        "min"
+      end
+
+      def get_definitions(schema)
+        type = get_schema_type(schema)
+        subtype = get_subtype_of(schema)
+
+        represent_str = "that represents the #{get_title(schema)} {{entity}}"
+        if subtype
+          represent_str = "that is a type of #{subtype} #{represent_str}"
+        end
+
+        definition = case type
+                     when "arm"
+                       "{{application object}} #{represent_str}"
+                     else
+                       "{{entity data type}} #{represent_str}"
+                     end
+        [Glossarist::DetailedDefinition.new(content: definition)]
+      end
+
+      def get_subtype_of(schema)
+        schema.entities.first&.subtype_of&.first&.id # rubocop:disable Style/SafeNavigationChainLength
+      end
+
+      # get entities remarks and remark items with id `__note` as notes
+      def get_notes(schema, schema_domain) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
+        notes = schema.entities&.map do |entity|
+          [
+            entity.remarks,
+            entity.remark_items&.select do |ri|
+              ri.id == "__note"
+            end&.map(&:remarks),
+          ]
+        end&.flatten&.compact
+
+        notes&.map do |note|
+          Glossarist::DetailedDefinition.new(
+            content: convert_express_xref(note, schema_domain),
+          )
+        end
+      end
+
+      # get entities remark items with id `__example` as examples
+      def get_examples(schema, schema_domain) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
+        examples = schema.entities&.map do |entity|
+          entity.remark_items&.select do |ri|
+            ri.id == "__example"
+          end&.map(&:remarks)
+        end&.flatten&.compact
+
+        examples&.map do |example|
+          Glossarist::DetailedDefinition.new(
+            content: convert_express_xref(example, schema_domain),
+          )
+        end
+      end
+
+      def convert_express_xref(content, schema_domain)
+        content.gsub(/<<express:(.*),(.*)>>/) do
+          "{{<#{schema_domain}>" \
+            "#{Regexp.last_match(1).split('.').last},#{Regexp.last_match(2)}}}"
+        end
+      end
+
+      def get_concept_filename(concept)
+        identifier = concept["data"]["identifier"]
+        "#{sanitize_string(identifier)}.yaml"
+      end
+
+      def sanitize_string(str)
+        str.gsub(" ", "_").gsub("/", "_").gsub(":", "_")
+      end
+    end
+  end
+end

data/lib/suma/cli/generate_schemas.rb

@@ -0,0 +1,141 @@
+require "thor"
+require_relative "../thor_ext"
+require "fileutils"
+require "yaml"
+require "pathname"
+
+module Suma
+  module Cli
+    # GenerateSchemas command to generate Schemas YAML by Metanorma YAML
+    class GenerateSchemas < Thor
+      desc "generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE",
+           "Generate EXPRESS schema manifest file from Metanorma site manifest"
+      option :exclude_paths, type: :string, default: nil, aliases: "-e",
+                             desc: "Exclude schemas paths by pattern " \
+                                   "(e.g. `*_lf.exp`)"
+
+      YAML_FILE_EXTENSIONS = [".yaml", ".yml"].freeze
+
+      def generate_schemas(metanorma_manifest_file, schema_manifest_file) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        metanorma_manifest_file = File.expand_path(metanorma_manifest_file)
+
+        unless File.exist?(metanorma_manifest_file)
+          raise Errno::ENOENT, "Specified file `#{metanorma_manifest_file}` " \
+                               "not found."
+        end
+
+        unless File.file?(metanorma_manifest_file)
+          raise ArgumentError, "Specified path `#{metanorma_manifest_file}` " \
+                               "is not a file."
+        end
+
+        [metanorma_manifest_file, schema_manifest_file].each do |file|
+          if !YAML_FILE_EXTENSIONS.include?(File.extname(file))
+            raise ArgumentError, "Specified file `#{file}` is not a YAML file."
+          end
+        end
+
+        run(
+          metanorma_manifest_file, schema_manifest_file,
+          exclude_paths: options[:exclude_paths]
+        )
+      end
+
+      private
+
+      def run(metanorma_manifest_file, schema_manifest_file, exclude_paths: nil)
+        metanorma_data = load_yaml(metanorma_manifest_file)
+        collection_files = metanorma_data["metanorma"]["source"]["files"]
+        manifest_files = load_manifest_files(collection_files)
+        all_schemas = load_project_schemas(manifest_files, exclude_paths,
+                                           schema_manifest_file)
+        all_schemas["schemas"] = all_schemas["schemas"].sort.to_h
+        output_data(all_schemas, schema_manifest_file)
+      end
+
+      def output_data(all_schemas, path)
+        puts "Writing the Schemas YAML file to #{File.expand_path(path)}..."
+        # debug use only
+        # puts all_schemas.to_yaml
+        File.write(File.expand_path(path), all_schemas.to_yaml)
+        puts "Writing the Schemas YAML file to #{File.expand_path(path)}...Done"
+      end
+
+      def load_yaml(file_path)
+        YAML.safe_load(
+          File.read(file_path, encoding: "UTF-8"),
+          permitted_classes: [Date, Time, Symbol],
+          permitted_symbols: [],
+          aliases: true,
+        )
+      end
+
+      def load_manifest_files(collection_files)
+        manifest_files = collection_files.map do |c|
+          collection_data = load_yaml(c)
+          collection_data["manifest"]["docref"].map { |docref| docref["file"] }
+        end
+        manifest_files.flatten
+      end
+
+      def load_project_schemas( # rubocop:disable Metrics/AbcSize
+        manifest_files, exclude_paths, schema_manifest_file
+      )
+        all_schemas = { "schemas" => {} }
+
+        manifest_files.each do |file|
+          # load schemas.yaml from the location of the collection.yml file
+          schemas_file_path = File.expand_path(
+            file.gsub("collection.yml", "schemas.yaml"),
+          )
+          unless File.exist?(schemas_file_path)
+            puts "Schemas file not found: #{schemas_file_path}"
+            next
+          end
+
+          schemas_data = load_yaml(schemas_file_path)
+
+          if schemas_data["schemas"]
+            schemas_data["schemas"] = fix_path(
+              schemas_data,
+              schemas_file_path,
+              schema_manifest_file,
+            )
+            all_schemas["schemas"].merge!(schemas_data["schemas"])
+          end
+
+          if exclude_paths
+            all_schemas["schemas"].delete_if do |_key, value|
+              value["path"].match?(
+                Regexp.new(exclude_paths.gsub("*", "(.*){1,999}")),
+              )
+            end
+          end
+        end
+
+        all_schemas
+      end
+
+      def fix_path(schemas_data, schemas_file_path, schema_manifest_file) # rubocop:disable Metrics/AbcSize
+        schema_manifest_path = File.expand_path(schema_manifest_file, Dir.pwd)
+
+        schemas_data["schemas"].each do |key, value|
+          # resolve the path in the schema file by the path in the schemas.yaml
+          path_in_schema = File.expand_path(
+            value["path"],
+            File.dirname(schemas_file_path),
+          )
+
+          # calculate the relative path from the schema manifest file
+          fixed_path = Pathname.new(path_in_schema).relative_path_from(
+            Pathname.new(File.dirname(schema_manifest_path)),
+          )
+
+          { key => value.merge!("path" => fixed_path.to_s) }
+        end
+
+        schemas_data["schemas"]
+      end
+    end
+  end
+end

data/lib/suma/cli.rb

@@ -23,6 +23,16 @@ module Suma
         Cli::Build.start
       end
 
+      desc "generate_schemas METANORMA_MANIFEST_FILE SCHEMA_MANIFEST_FILE",
+           "Generate EXPRESS schema manifest file from Metanorma site manifest"
+      option :exclude_paths, type: :string, default: nil, aliases: "-e",
+                             desc: "Exclude schemas paths by pattern " \
+                                   "(e.g. `*_lf.exp`)"
+      def generate_schemas(_metanorma_manifest_file, _schema_manifest_file)
+        require_relative "cli/generate_schemas"
+        Cli::GenerateSchemas.start
+      end
+
       desc "reformat EXPRESS_FILE_PATH",
            "Reformat EXPRESS files"
       option :recursive, type: :boolean, default: false, aliases: "-r",
@@ -33,6 +43,16 @@ module Suma
         Cli::Reformat.start
       end
 
+      desc "extract_terms SCHEMA_MANIFEST_FILE GLOSSARIST_OUTPUT_PATH",
+           "Extract terms from SCHEMA_MANIFEST_FILE into " \
+           "Glossarist v2 format"
+      option :language_code, type: :string, default: "eng", aliases: "-l",
+                             desc: "Language code for the Glossarist"
+      def extract_terms(_schema_manifest_file, _glossarist_output_path)
+        require_relative "cli/extract_terms"
+        Cli::ExtractTerms.start
+      end
+
       desc "validate SUBCOMMAND ...ARGS", "Validate express documents"
       subcommand "validate", Cli::Validate
 

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.1.13"
+  VERSION = "0.1.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-05-24 00:00:00.000000000 Z
+date: 2025-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -134,6 +134,8 @@ files:
 - lib/suma.rb
 - lib/suma/cli.rb
 - lib/suma/cli/build.rb
+- lib/suma/cli/extract_terms.rb
+- lib/suma/cli/generate_schemas.rb
 - lib/suma/cli/reformat.rb
 - lib/suma/cli/validate.rb
 - lib/suma/cli/validate_ascii.rb