suma 0.2.6 → 0.4.0

data/lib/suma/express_reformatter.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Suma
+  # Reformats EXPRESS source into suma's canonical form: comment blocks
+  # (the +(*"...\n*)+ remarks that EXPRESS uses for documentation) are
+  # extracted from their inline positions and appended to the end of
+  # the file, and excess blank lines are collapsed.
+  #
+  # Pure content transformation — no I/O, no Thor, no filesystem. The
+  # CLI adapter handles reading from and writing to disk.
+  #
+  # The transform is idempotent: reformatting an already-reformatted
+  # document returns +changed?: false+.
+  #
+  # Comment extraction uses +String#index+ rather than a single m-mode
+  # regex. The original +/\(\*"(.*?)\n\*\)/m+ is correct functionally
+  # but is polynomial-time on adversarial input (CodeQL
+  # +rb/polynomial-redos+). Scanning for the start marker, then for
+  # the next terminator, is unambiguously linear.
+  module ExpressReformatter
+    Result = Struct.new(:content, :changed?, keyword_init: true) do
+      def content_or_nil
+        changed? ? content : nil
+      end
+    end
+
+    COMMENT_START = '(*"'
+    COMMENT_END = "\n*)"
+    BLANK_RUN_PATTERN = /(\n\n+)/
+    NEWLINE_RUN_PATTERN = /(\n+)/
+
+    module_function
+
+    def call(content)
+      comments = extract_comments(content)
+      return Result.new(content: content, changed?: false) if comments.empty?
+
+      without_comments = strip_comments(content)
+      new_comments = comments.map { |c| "#{COMMENT_START}#{c}#{COMMENT_END}" }
+        .join("\n\n")
+      new_content = "#{without_comments}\n\n#{new_comments}\n"
+      new_content = new_content.gsub(BLANK_RUN_PATTERN, "\n\n")
+
+      changed = normalised_compare(content, new_content) != 0
+      Result.new(content: new_content, changed?: changed)
+    end
+
+    def extract_comments(content)
+      comments = []
+      pos = 0
+      while (start_idx = content.index(COMMENT_START, pos))
+        end_idx = content.index(COMMENT_END, start_idx + COMMENT_START.length)
+        break unless end_idx
+
+        comments << content[(start_idx + COMMENT_START.length)...end_idx]
+        pos = end_idx + COMMENT_END.length
+      end
+      comments
+    end
+
+    def strip_comments(content)
+      return content unless content.index(COMMENT_START, 0)
+
+      stripped = +""
+      last_end = 0
+      each_comment_range(content) do |range|
+        stripped << content[range[:pre_start]...range[:start]]
+        last_end = range[:end_exclusive]
+      end
+      stripped << content[last_end..]
+      stripped.force_encoding(content.encoding)
+    end
+
+    def each_comment_range(content)
+      return enum_for(:each_comment_range, content) unless block_given?
+
+      pos = 0
+      while (start_idx = content.index(COMMENT_START, pos))
+        end_idx = content.index(COMMENT_END, start_idx + COMMENT_START.length)
+        break unless end_idx
+
+        yield pre_start: pos, start: start_idx,
+              end_exclusive: end_idx + COMMENT_END.length
+        pos = end_idx + COMMENT_END.length
+      end
+    end
+
+    def normalised_compare(left, right)
+      left.gsub(NEWLINE_RUN_PATTERN, "\n") <=> right.gsub(NEWLINE_RUN_PATTERN, "\n")
+    end
+
+    private_class_method :normalised_compare
+  end
+end

data/lib/suma/express_schema.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "utils"
 require "fileutils"
 require "expressir"
 

data/lib/suma/jsdai/figure.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "figure_xml"
-require_relative "figure_image"
-
 module Suma
   module Jsdai
     # Main class for JSDAI figure conversion

data/lib/suma/jsdai.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 module Suma
   module Jsdai
+    autoload :Figure,      "suma/jsdai/figure"
+    autoload :FigureImage, "suma/jsdai/figure_image"
+    autoload :FigureXml,   "suma/jsdai/figure_xml"
   end
 end
-
-require_relative "jsdai/figure"

data/lib/suma/link_validation.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "expressir"
+
+module Suma
+  # Deep module behind the EXPRESS cross-reference validation pipeline.
+  #
+  # Interface: paths in, +Result+ out. The CLI is a thin adapter that
+  # constructs this object and prints the result.
+  #
+  # Owns: loading the schemas manifest, discovering + reading .adoc and
+  # .exp files, extracting express cross-reference links, loading parsed
+  # schemas into a +SchemaIndex+, delegating to +LinkValidator+ for the
+  # actual resolution, and writing the summary file.
+  #
+  # Does not own: presentation (use +LinkValidation.generate_summary+),
+  # command-line argument parsing (the CLI adapter does that), or the
+  # link-resolution rules themselves (that is +LinkValidator+'s job).
+  class LinkValidation
+    EXPRESS_LINK_PATTERN = /<<express:([^,>]+)(?:,[^>]+)?>>/
+
+    attr_reader :schemas_file, :documents_path, :output_file,
+                :progress, :logger
+
+    def initialize(schemas_file:, documents_path:, output_file:,
+                   progress: NullProgress.new, logger: Utils)
+      @schemas_file = Pathname.new(schemas_file).expand_path
+      @documents_path = Pathname.new(documents_path).expand_path
+      @output_file = output_file && Pathname.new(output_file).expand_path
+      @progress = progress
+      @logger = logger
+    end
+
+    def call
+      config = load_schemas_config
+      exp_files = collect_schema_paths(config)
+      adoc_files = find_adoc_files
+      links_by_file = extract_links(adoc_files + exp_files)
+      unresolved = validate_links(config, links_by_file)
+      result = Result.new(
+        adoc_count: adoc_files.size,
+        exp_count: exp_files.size,
+        total_links: links_by_file.values.sum(&:size),
+        unresolved: unresolved,
+      )
+      write_summary(result) if output_file
+      result
+    end
+
+    def self.generate_summary(result)
+      lines = []
+      lines << "Validation complete. Checked #{result.total_links} links."
+      if result.success?
+        lines << "✅ All links resolved successfully!"
+      else
+        lines << "❌ Found #{result.unresolved.size} unresolved links:"
+        result.unresolved.each { |issue| lines << format_issue(issue) }
+      end
+      lines.join("\n")
+    end
+
+    def self.format_issue(issue)
+      "#{issue.file}:#{issue.line} - " \
+        "<<express:#{issue.link}>> - #{issue.reason}"
+    end
+    private_class_method :format_issue
+
+    # Default no-op progress adapter. Callers that want a real progress
+    # bar pass an object responding to +#start(title, total)+ and
+    # +#increment+; this satisfies the same interface without forcing
+    # the dependency.
+    class NullProgress
+      def start(_title, _total); end
+
+      def increment; end
+    end
+
+    Result = Struct.new(
+      :adoc_count,
+      :exp_count,
+      :total_links,
+      :unresolved,
+      keyword_init: true,
+    ) do
+      def success?
+        unresolved.empty?
+      end
+    end
+
+    private
+
+    def load_schemas_config
+      Expressir::SchemaManifest.from_yaml(File.read(schemas_file))
+        .tap { |c| c.set_initial_path(schemas_file.to_s) }
+    rescue StandardError => e
+      raise Error, "Error loading schemas file: #{e.message}"
+    end
+
+    def collect_schema_paths(schemas_config)
+      schemas_config.schemas.filter_map(&:path)
+    end
+
+    def find_adoc_files
+      Dir.glob(documents_path.join("**", "*.adoc").to_s)
+    end
+
+    def extract_links(files)
+      links_by_file = {}
+      progress.start("Processing files", files.size)
+      files.each do |file|
+        links = extract_links_from_file(file)
+        links_by_file[file] = links if links&.any?
+      end
+      links_by_file
+    end
+
+    def extract_links_from_file(file)
+      progress.increment
+      content = File.read(file)
+      content.scan(EXPRESS_LINK_PATTERN).flatten.uniq
+    rescue StandardError => e
+      logger.log "Warning: Could not read file #{file}: #{e.message}"
+      nil
+    end
+
+    def validate_links(schemas_config, links_by_file)
+      paths_by_id = schemas_config.schemas.to_h { |s| [s.id, s.path] }
+      progress.start("Loading schemas", paths_by_id.size)
+      repo = Expressir::Express::Parser.from_files(paths_by_id.values) do |*_args|
+        progress.increment
+      end
+      index = SchemaIndex.new(repo)
+      LinkValidator.new(index).validate(links_by_file)
+    end
+
+    def write_summary(result)
+      FileUtils.mkdir_p(output_file.dirname)
+      File.write(output_file, self.class.generate_summary(result))
+    rescue StandardError => e
+      logger.log "Error writing to output file: #{e.message}"
+    end
+  end
+end

data/lib/suma/link_validator.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative "schema_index"
 require "expressir"
+require "suma/link_validation"
 
 module Suma
-  LinkValidationResult = Struct.new(:file, :line, :link, :reason, keyword_init: true)
+  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
+                                    keyword_init: true)
 
   class LinkValidator
     def initialize(index)
@@ -28,7 +29,7 @@ module Suma
       content = File.read(file)
       index = {}
       content.lines.each_with_index do |line, idx|
-        line.scan(/<<express:([^,>]+)(?:,[^>]+)?>>/).flatten.each do |link|
+        line.scan(LinkValidation::EXPRESS_LINK_PATTERN).flatten.each do |link|
           index[link] ||= idx
         end
       end
@@ -93,18 +94,22 @@ module Suma
 
       return unless parts.size > 2
 
-      error = validate_deep_path(schema, element, parts[2..], file, line_idx, link)
+      error = validate_deep_path(schema, element, parts[2..], file, line_idx,
+                                 link)
       unresolved << error if error
     end
 
-    def validate_deep_path(schema, element, path_parts, file, line_idx, full_link)
+    def validate_deep_path(schema, element, path_parts, file, line_idx,
+full_link)
       current = element
       current_path = "#{schema.id}.#{element.id}"
 
       path_parts.each do |part|
         case current
         when Expressir::Model::Declarations::Entity
-          attribute = current.attributes&.find { |a| a.id.downcase == part.downcase }
+          attribute = current.attributes&.find do |a|
+            a.id.downcase == part.downcase
+          end
 
           unless attribute
             return LinkValidationResult.new(
@@ -122,7 +127,9 @@ module Suma
           underlying = current.underlying_type
 
           if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
-            enum_value = underlying.items.find { |e| e.id.downcase == part.downcase }
+            enum_value = underlying.items.find do |e|
+              e.id.downcase == part.downcase
+            end
 
             unless enum_value
               return LinkValidationResult.new(
@@ -193,7 +200,9 @@ module Suma
         schema.procedures,
         schema.subtype_constraints,
       ].each do |collection|
-        element = collection&.find { |e| e.id.downcase == element_name.downcase }
+        element = collection&.find do |e|
+          e.id.downcase == element_name.downcase
+        end
         return element if element
       end
 

data/lib/suma/manifest_traverser.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "expressir"
+
+module Suma
+  # Tree-walking service over a CollectionManifest.
+  #
+  # ManifestTraverser owns all imperative operations that walk or mutate a
+  # manifest tree: finding schemas-only entries, expanding them into
+  # compiled-doc sub-trees, exporting a unified Expressir::SchemaManifest,
+  # and removing schemas-only sources after compilation.
+  #
+  # Each method takes the manifest passed at construction as its root;
+  # recursion instantiates a new traverser per child node so the surface
+  # stays instance-based and the data model (CollectionManifest) stays pure.
+  #
+  # Schema-config loading and doc-entry construction are delegated to
+  # SchemaDiscovery so this class owns traversal, not schema I/O.
+  class ManifestTraverser
+    attr_reader :manifest
+
+    def initialize(manifest)
+      @manifest = manifest
+    end
+
+    # Returns every entry (anywhere in the tree) whose `schemas_only` flag
+    # is truthy, plus +manifest+ itself if it is schemas-only.
+    def find_schemas_only
+      results = (manifest.entry || []).select(&:schemas_only)
+      results << manifest if manifest.schemas_only
+      results
+    end
+
+    # Recursively concatenate every nested schema_config into a single
+    # Expressir::SchemaManifest. The manifest's own schema_config (if set)
+    # seeds the result; otherwise an empty SchemaManifest is returned.
+    def export_schema_config(path)
+      export_config = manifest.schema_config || Expressir::SchemaManifest.new
+      return export_config unless manifest.entry
+
+      manifest.entry.each do |child|
+        child_config = self.class.new(child).export_schema_config(path)
+        export_config.concat(child_config) if child_config
+      end
+
+      export_config
+    end
+
+    # Expand schemas-only entries into compiled-doc sub-trees. If the
+    # manifest has no file, walks children via process_entry. Otherwise
+    # loads the schema_config (SchemaDiscovery), and if this entry is
+    # schemas-only, hides it from the output index and appends a new
+    # sub-collection that hosts the compiled docs.
+    def expand_schemas_only(schema_output_path)
+      return process_entry(schema_output_path) unless manifest.file
+
+      SchemaDiscovery.new(manifest).load_config
+
+      return process_entry(schema_output_path) unless manifest.schemas_only
+
+      manifest.index = false
+
+      added = SchemaDiscovery.new(manifest).build_added_manifest(schema_output_path)
+      [manifest, added]
+    end
+
+    # Drop every direct child whose `schemas_only` is truthy. Called after
+    # compilation so the schemas-only manifest file no longer shows up in
+    # the rendered collection.
+    def remove_schemas_only_sources
+      return unless manifest.entry
+
+      kept = manifest.entry.reject(&:schemas_only)
+      manifest.entry = kept
+    end
+
+    private
+
+    # Walk each child via expand_schemas_only, flattening the results back
+    # into manifest.entry.
+    def process_entry(schema_output_path)
+      return [manifest] unless manifest.entry
+
+      flattened = manifest.entry.each_with_object([]) do |child, acc|
+        acc.concat(self.class.new(child).expand_schemas_only(schema_output_path))
+      end
+
+      manifest.entry = flattened
+      [manifest]
+    end
+  end
+end

data/lib/suma/processor.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "schema_collection"
-require_relative "utils"
-require_relative "collection_config"
-require_relative "site_config"
 require "metanorma"
 
 module Suma
@@ -41,11 +37,12 @@ module Suma
       collection_config_path = site_config.metanorma.source.files.first
       collection_config = Suma::CollectionConfig.from_file(collection_config_path)
       collection_config.path = collection_config_path
-      collection_config.manifest.expand_schemas_only("schema_docs")
 
-      exported_schema_config = collection_config.manifest.export_schema_config(schemas_all_path)
-      exported_schema_config.path = schemas_all_path
+      traverser = ManifestTraverser.new(collection_config.manifest)
+      traverser.expand_schemas_only("schema_docs")
 
+      exported_schema_config = traverser.export_schema_config(schemas_all_path)
+      exported_schema_config.path = schemas_all_path
       exported_schema_config.to_file
 
       collection_config
@@ -72,7 +69,7 @@ module Suma
 
     def build_collection(collection_config, output_directory)
       new_collection_config_path = "collection-output.yaml"
-      collection_config.manifest.remove_schemas_only_sources
+      ManifestTraverser.new(collection_config.manifest).remove_schemas_only_sources
       collection_config.to_file(new_collection_config_path)
 
       collection = Metanorma::Collection.parse(new_collection_config_path)

data/lib/suma/register_manifest_generator.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "pathname"
+require "expressir"
+require "glossarist"
+
+module Suma
+  # Generates a Glossarist v3 register.yaml from an EXPRESS schema manifest.
+  #
+  # The schema manifest (schemas-smrl-part-2.yml) is the single source of
+  # truth for schema identities and paths. This class reads it, classifies
+  # each schema by type (resource/module), and emits a register.yaml with
+  # hierarchical sections and human-readable names.
+  #
+  # Architecture:
+  #   - Classification: delegates to ExpressSchema::Type (DRY)
+  #   - Naming: delegates to SchemaNaming (OCP — extend naming without
+  #     touching this class)
+  #   - Sections: built from Glossarist::Section models (model-driven)
+  #   - URN semantics: delegates to Suma::Urn (OCP)
+  #
+  # @example
+  #   generator = RegisterManifestGenerator.new(
+  #     "schemas-smrl-part-2.yml",
+  #     urn: "urn:iso:std:iso:10303:-2:ed-2:en:tech:*",
+  #     id: "iso10303-2-express",
+  #     ref: "ISO 10303-2 EXPRESS Concepts",
+  #   )
+  #   generator.generate  # writes register.yaml
+  class RegisterManifestGenerator
+    DEFAULT_OWNER = "ISO/TC 184/SC 4"
+    DEFAULT_STATUS = "current"
+    DEFAULT_ORDERING = "systematic"
+    DEFAULT_SCHEMA_TYPE = "glossarist"
+    DEFAULT_SCHEMA_VERSION = "3"
+
+    # @param schema_manifest_file [String] path to schemas-smrl-part-2.yml
+    # @param output_path [String] directory to write register.yaml
+    # @param urn [String] base URN for the dataset
+    # @param id [String] dataset identifier
+    # @param ref [String] human-readable reference label
+    # @param language_code [String] language for section names
+    # @param owner [String] dataset owner organisation
+    def initialize(schema_manifest_file, output_path, urn:, id:, ref:,
+                   language_code: "eng", owner: DEFAULT_OWNER)
+      @schema_manifest_file = File.expand_path(schema_manifest_file)
+      @output_path = output_path
+      @urn = Suma::Urn.new(urn)
+      @id = id
+      @ref = ref
+      @language_code = language_code
+      @owner = owner
+    end
+
+    # Generate and write register.yaml.
+    #
+    # @return [Glossarist::DatasetRegister] the generated register
+    def generate
+      validate_inputs
+      schemas = load_schemas
+      register = build_register(schemas)
+
+      FileUtils.mkdir_p(@output_path)
+      output_file = File.join(@output_path, "register.yaml")
+      File.write(output_file, register.to_yaml)
+      Utils.log "Generated register.yaml: #{output_file}"
+      Utils.log "  #{schemas.length} schemas in #{register.sections.length} categories"
+
+      register
+    end
+
+    private
+
+    # Verify that the manifest path exists, is a regular file, and contains
+    # at least one schema entry. Called from {#generate} so both the CLI
+    # shell and direct construction get the same validation behavior.
+    #
+    # @raise [Errno::ENOENT] when the manifest path is missing or not a file
+    # @raise [ArgumentError] when the manifest is empty
+    def validate_inputs
+      unless File.exist?(@schema_manifest_file)
+        raise Errno::ENOENT, "Specified SCHEMA_MANIFEST_FILE " \
+                             "`#{@schema_manifest_file}` not found."
+      end
+      unless File.file?(@schema_manifest_file)
+        raise Errno::ENOENT, "Specified SCHEMA_MANIFEST_FILE " \
+                             "`#{@schema_manifest_file}` is not a file."
+      end
+    end
+
+    # Load all schemas from the manifest file as SchemaManifestEntry models.
+    #
+    # @return [Array<Expressir::SchemaManifestEntry>]
+    def load_schemas
+      manifest = Expressir::SchemaManifest.from_file(@schema_manifest_file)
+      schemas = manifest.schemas.sort_by { |s| s.id.downcase }
+      if schemas.empty?
+        raise ArgumentError, "No schemas found in manifest " \
+                             "`#{@schema_manifest_file}`."
+      end
+      schemas
+    end
+
+    # Build a fully-populated DatasetRegister model.
+    #
+    # @param schemas [Array<Expressir::SchemaManifestEntry>]
+    # @return [Glossarist::DatasetRegister]
+    def build_register(schemas)
+      Glossarist::DatasetRegister.new(
+        schema_type: DEFAULT_SCHEMA_TYPE,
+        schema_version: DEFAULT_SCHEMA_VERSION,
+        id: @id,
+        ref: @ref,
+        urn: @urn.to_s,
+        urn_aliases: @urn.aliases,
+        status: DEFAULT_STATUS,
+        owner: @owner,
+        languages: [@language_code],
+        ordering: DEFAULT_ORDERING,
+        sections: build_sections(schemas),
+      )
+    end
+
+    # Build a list of top-level sections, one per category, in
+    # SchemaCategory::ALL declaration order. Categories with no
+    # schemas are omitted.
+    #
+    # @param schemas [Array<Expressir::SchemaManifestEntry>]
+    # @return [Array<Glossarist::Section>]
+    def build_sections(schemas)
+      groups = schemas.group_by do |s|
+        SchemaCategory.for_schema(id: s.id, path: s.path)
+      end
+
+      SchemaCategory::ALL.filter_map do |category|
+        children = groups[category] || []
+        next if children.empty?
+
+        Glossarist::Section.new(
+          id: category.id,
+          names: { @language_code => category.label },
+          children: children.map { |s| build_section(s) },
+        )
+      end
+    end
+
+    # Build a single leaf section from a schema descriptor.
+    #
+    # @param schema [Expressir::SchemaManifestEntry]
+    # @return [Glossarist::Section]
+    def build_section(schema)
+      Glossarist::Section.new(
+        id: schema.id,
+        names: {
+          @language_code => SchemaNaming.prefixed_name(
+            schema.id, path: schema.path
+          ),
+        },
+      )
+    end
+  end
+end