suma 0.4.0 → 0.5.1

data/lib/suma/link_validator/step.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "expressir"
+
+module Suma
+  class LinkValidator
+    # Per-node-type navigation strategies for +validate_deep_path+.
+    #
+    # Each Step handles one EXPRESS construct (entity, type, ...) and
+    # knows how to walk one path segment from it. The dispatcher in
+    # LinkValidator picks the right Step by +handles?+.
+    #
+    # Adding a new navigable construct means adding a new Step class
+    # and registering it in REGISTRY — no edits to existing Steps or
+    # to validate_deep_path (open/closed principle).
+    module Step
+      Context = Struct.new(:file, :line, :link, :schema, :path,
+                           keyword_init: true)
+
+      # Common failure-result builder. Step subclasses call this to
+      # surface an unresolved link.
+      module Failure
+        def failure(reason, context)
+          LinkValidationResult.new(
+            file: context.file,
+            line: context.line + 1,
+            link: context.link,
+            reason: reason,
+          )
+        end
+      end
+
+      # Navigates one path segment from an Entity by attribute name.
+      class Entity
+        include Failure
+
+        def self.handles?(node)
+          node.is_a?(Expressir::Model::Declarations::Entity)
+        end
+
+        def navigate(node, part, context)
+          attribute = node.attributes&.find { |a| a.id.downcase == part.downcase }
+          unless attribute
+            return failure("Attribute '#{part}' not found in entity '#{context.path}'",
+                           context)
+          end
+
+          attribute
+        end
+      end
+
+      # Navigates one path segment from a Type by either enumeration
+      # value (for enum types) or by re-targeting to the underlying
+      # named type (for non-enum types).
+      class Type
+        include Failure
+
+        PRIMITIVE_TYPE_NAMES = %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
+                                  LOGICAL].freeze
+
+        def self.handles?(node)
+          node.is_a?(Expressir::Model::Declarations::Type)
+        end
+
+        def navigate(node, part, context)
+          underlying = node.underlying_type
+          unless underlying
+            return failure("Cannot navigate deeper from type '#{context.path}'",
+                           context)
+          end
+
+          if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
+            navigate_enum(underlying, part, context)
+          else
+            navigate_underlying(underlying, context)
+          end
+        end
+
+        private
+
+        def navigate_enum(underlying, part, context)
+          enum_value = underlying.items.find { |e| e.id.downcase == part.downcase }
+          unless enum_value
+            return failure("Enumeration value '#{part}' not found in type '#{context.path}'",
+                           context)
+          end
+
+          enum_value
+        end
+
+        def navigate_underlying(underlying, context)
+          base_type = resolve_base_type(context.schema, underlying)
+          unless base_type
+            return failure("Base type not found for '#{context.path}'",
+                           context)
+          end
+
+          base_type
+        end
+
+        def resolve_base_type(schema, type_ref)
+          return nil if PRIMITIVE_TYPE_NAMES.include?(type_ref.to_s.upcase)
+          return find_schema_element(schema, type_ref) if type_ref.is_a?(String)
+
+          type_ref if type_ref.is_a?(Expressir::Model::ModelElement)
+        end
+
+        def find_schema_element(schema, name)
+          SchemaElementLookup.find(schema, name)
+        end
+      end
+
+      # Lookup helper shared across Step classes that need to resolve
+      # a named schema element (used by Type for underlying-type
+      # resolution). Lives behind its own module so Steps don't reach
+      # into LinkValidator's private methods.
+      module SchemaElementLookup
+        def self.find(schema, name)
+          collections(schema).each do |collection|
+            element = collection&.find { |e| e.id.downcase == name.downcase }
+            return element if element
+          end
+          nil
+        end
+
+        def self.collections(schema)
+          [
+            schema.entities,
+            schema.types,
+            schema.constants,
+            schema.functions,
+            schema.rules,
+            schema.procedures,
+            schema.subtype_constraints,
+          ]
+        end
+        private_class_method :collections
+      end
+
+      REGISTRY = [Entity, Type].freeze
+
+      # Returns the Step class that handles +node+, or +nil+ if no
+      # registered Step matches.
+      def self.for(node)
+        REGISTRY.find { |step| step.handles?(node) }
+      end
+    end
+  end
+end

data/lib/suma/link_validator.rb

@@ -8,6 +8,8 @@ module Suma
                                     keyword_init: true)
 
   class LinkValidator
+    autoload :Step, "suma/link_validator/step"
+
     def initialize(index)
       @index = index
     end
@@ -100,113 +102,34 @@ module Suma
     end
 
     def validate_deep_path(schema, element, path_parts, file, line_idx,
-full_link)
+                           full_link)
       current = element
       current_path = "#{schema.id}.#{element.id}"
+      context = Step::Context.new(file: file, line: line_idx, link: full_link,
+                                  schema: schema, path: current_path)
 
       path_parts.each do |part|
-        case current
-        when Expressir::Model::Declarations::Entity
-          attribute = current.attributes&.find do |a|
-            a.id.downcase == part.downcase
-          end
-
-          unless attribute
-            return LinkValidationResult.new(
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Attribute '#{part}' not found in entity '#{current_path}'",
-            )
-          end
-
-          current = attribute
-          current_path += ".#{part}"
-
-        when Expressir::Model::Declarations::Type
-          underlying = current.underlying_type
-
-          if underlying.is_a?(Expressir::Model::DataTypes::Enumeration)
-            enum_value = underlying.items.find do |e|
-              e.id.downcase == part.downcase
-            end
-
-            unless enum_value
-              return LinkValidationResult.new(
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Enumeration value '#{part}' not found in type '#{current_path}'",
-              )
-            end
-
-            current = enum_value
-            current_path += ".#{part}"
-
-          elsif underlying
-            base_type = find_base_type(schema, underlying)
-
-            unless base_type
-              return LinkValidationResult.new(
-                file: file,
-                line: line_idx + 1,
-                link: full_link,
-                reason: "Base type not found for '#{current_path}'",
-              )
-            end
-
-            current = base_type
-
-          else
-            return LinkValidationResult.new(
-              file: file,
-              line: line_idx + 1,
-              link: full_link,
-              reason: "Cannot navigate deeper from type '#{current_path}'",
-            )
-          end
+        step = Step.for(current)
+        return unsupported_step_failure(context) unless step
 
-        else
-          return LinkValidationResult.new(
-            file: file,
-            line: line_idx + 1,
-            link: full_link,
-            reason: "Cannot navigate deeper from '#{current_path}'",
-          )
-        end
+        outcome = step.new.navigate(current, part, context)
+        return outcome if outcome.is_a?(LinkValidationResult)
+
+        current = outcome
+        current_path += ".#{part}"
+        context.path = current_path
       end
 
       nil
     end
 
-    def find_base_type(schema, type_ref)
-      return nil if %w[INTEGER REAL STRING BOOLEAN NUMBER BINARY
-                       LOGICAL].include?(type_ref.to_s.upcase)
-
-      if type_ref.is_a?(String)
-        find_schema_element(schema, type_ref)
-      elsif type_ref.is_a?(Expressir::Model::ModelElement)
-        type_ref
-      end
-    end
-
-    def find_schema_element(schema, element_name)
-      [
-        schema.entities,
-        schema.types,
-        schema.constants,
-        schema.functions,
-        schema.rules,
-        schema.procedures,
-        schema.subtype_constraints,
-      ].each do |collection|
-        element = collection&.find do |e|
-          e.id.downcase == element_name.downcase
-        end
-        return element if element
-      end
-
-      nil
+    def unsupported_step_failure(context)
+      LinkValidationResult.new(
+        file: context.file,
+        line: context.line + 1,
+        link: context.link,
+        reason: "Cannot navigate deeper from '#{context.path}'",
+      )
     end
   end
 end

data/lib/suma/note_processor.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "glossarist"
+
+module Suma
+  # Cleans raw EXPRESS entity remarks into a list of
+  # +Glossarist::V3::DetailedDefinition+ notes.
+  #
+  # Pipeline:
+  #   1. trim to first sentence / first complete list
+  #   2. convert express xrefs to URN mentions
+  #   3. drop redundant "X is a type of {{...}}" opening notes
+  #   4. drop notes containing image references or <<...>> blocks
+  #   5. drop notes that duplicate the first definition
+  #
+  # Pure: no I/O, no schema knowledge. The schema domain label and
+  # URN composition are injected via +xref_to_mention+ so this module
+  # stays independent of +Suma::Urn+ and the surrounding extractor.
+  class NoteProcessor
+    REDUNDANT_NOTE_REGEX =
+      %r{
+        ^An?                   # Starts with "A" or "An"
+        \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
+        (\sa|\san)?            # Optional " a" or " an"
+        \s\{\{[^\}]*\}\}       # Text in double curly braces
+        \s*?\.?$               # Optional whitespace and period at the end
+      }x
+
+    IMAGE_REF_SUBSTRING = "image::"
+    DOUBLE_ANGLE_PATTERN = /<<(.*?){1,999}>>/
+    SEE_TAIL_PATTERN = /\s+\(see(.*?){1,999}\)/
+    XREF_WITH_DISPLAY = /<<express:([\w.]+),([\w. ][\w. ]*)>>/
+    XREF_WITHOUT_DISPLAY = /<<express:([\w.]+)>>/
+    INLINE_COMMENT_PATTERN = /\n\/\/.*?\n/
+    BULLET_LIST_PATTERN = /^\s*[*\-+]\s+/
+    NUMBERED_LIST_PATTERN = /^\s*\d+\.\s+/
+    CONTINUATION_PLUS = /^\+\s*$/
+    CONTINUATION_DASHES = /^--\s*$/
+    INDENTED_PATTERN = /^\s{2,}/
+    SENTENCE_BREAK_NEWLINE = /\.\n/
+    SENTENCE_BREAK_SPACE = /\. /
+
+    attr_reader :remarks, :definitions, :xref_to_mention
+
+    def initialize(remarks, definitions:, xref_to_mention:)
+      @remarks = remarks
+      @definitions = definitions
+      @xref_to_mention = xref_to_mention
+    end
+
+    def self.call(remarks, definitions:, xref_to_mention:)
+      new(remarks, definitions: definitions, xref_to_mention: xref_to_mention).call
+    end
+
+    def call
+      notes = build_initial_notes
+      notes = only_keep_first_sentence(notes)
+      notes = remove_see_content(notes)
+      notes = remove_redundant_note(notes)
+      notes = remove_invalid_references(notes)
+      compare_with_definitions(notes, definitions)
+    end
+
+    private
+
+    def build_initial_notes
+      trimmed = trim_definition(remarks)
+      return [] unless trimmed && !trimmed.empty?
+
+      [Glossarist::V3::DetailedDefinition.new(content: trimmed)]
+    end
+
+    def trim_definition(definition)
+      return nil if definition.nil? || definition.empty?
+
+      definition_str = array_to_paragraphs(definition)
+      return nil if definition_str.empty?
+
+      paragraphs = definition_str.split("\n\n")
+      combine_paragraphs(paragraphs)
+    end
+
+    def array_to_paragraphs(definition)
+      definition.is_a?(Array) ? definition.join("\n\n") : definition.to_s
+    end
+
+    def combine_paragraphs(paragraphs)
+      combined = first_block(paragraphs)
+      combined = "#{combined}\n"
+      combined = combined.gsub(INLINE_COMMENT_PATTERN, "\n")
+      combined.strip!
+      convert_xref(combined)
+    end
+
+    def first_block(paragraphs)
+      first_paragraph = paragraphs.first
+      return apply_first_sentence_logic(first_paragraph) unless list_header?(paragraphs)
+
+      complete_list = extract_complete_list(paragraphs, 1)
+      "#{first_paragraph}\n\n#{complete_list}"
+    end
+
+    def list_header?(paragraphs)
+      paragraphs.length > 1 &&
+        paragraphs.first.end_with?(":") &&
+        starts_with_list?(paragraphs[1])
+    end
+
+    def apply_first_sentence_logic(paragraph)
+      new_content = paragraph
+        .split(SENTENCE_BREAK_NEWLINE).first.strip
+        .split(SENTENCE_BREAK_SPACE).first.strip
+
+      new_content.end_with?(".") ? new_content : "#{new_content}."
+    end
+
+    def extract_complete_list(paragraphs, start_index)
+      return paragraphs[start_index] if start_index >= paragraphs.length
+
+      reduce_list_paragraphs(paragraphs, start_index)
+    end
+
+    def reduce_list_paragraphs(paragraphs, start_index)
+      combined = paragraphs[start_index].dup
+      state = ContinuationState.new(combined.include?("--") && !combined.match?(/--.*--/m))
+
+      ((start_index + 1)...paragraphs.length).each do |i|
+        break unless apply_paragraph_step?(combined, paragraphs[i], state)
+      end
+
+      combined
+    end
+
+    # Returns true if iteration should continue, false to stop.
+    def apply_paragraph_step?(combined, para, state)
+      case classify_paragraph(para, state)
+      when :continuation_toggle
+        state.toggle!
+        append!(combined, para)
+      when :continuation
+        append!(combined, para)
+      when :list_member
+        append!(combined, para)
+        state.toggle! if opens_continuation_block?(para)
+      else
+        return false
+      end
+      true
+    end
+
+    def append!(combined, para)
+      combined << "\n\n#{para}"
+    end
+
+    def opens_continuation_block?(para)
+      para.include?("--") && !para.match?(/--.*--/m)
+    end
+
+    def classify_paragraph(para, state)
+      return :continuation_toggle if para.match?(CONTINUATION_DASHES) || para.end_with?("--")
+      return :continuation if state.in_block?
+      return :list_member if starts_with_list?(para) || list_continuation?(para)
+
+      :stop
+    end
+
+    # Mutable flag tracking whether we are inside a "-- ... --"
+    # continuation block while walking paragraphs.
+    class ContinuationState
+      def initialize(in_block)
+        @in_block = in_block
+      end
+
+      def in_block?
+        @in_block
+      end
+
+      def toggle!
+        @in_block = !@in_block
+      end
+    end
+    private_constant :ContinuationState
+
+    def list_continuation?(content)
+      return false if content.nil? || content.empty?
+
+      content.match?(CONTINUATION_PLUS) ||
+        content.match?(CONTINUATION_DASHES) ||
+        content.match?(INDENTED_PATTERN) ||
+        content.start_with?("which", "where", "that")
+    end
+
+    def starts_with_list?(content)
+      return false if content.nil? || content.empty?
+
+      content.match?(BULLET_LIST_PATTERN) || content.match?(NUMBERED_LIST_PATTERN)
+    end
+
+    def only_keep_first_sentence(notes)
+      notes.each do |note|
+        next unless note&.content
+        next if preserve_complete_structure?(note.content)
+
+        new_content = note.content
+          .split(SENTENCE_BREAK_NEWLINE).first.strip
+          .split(SENTENCE_BREAK_SPACE).first.strip
+        note.content = new_content.end_with?(".") ? new_content : "#{new_content}."
+      end
+    end
+
+    def preserve_complete_structure?(content)
+      return false if content.nil? || content.empty?
+
+      lines = content.split("\n")
+      return false unless first_line_is_header?(lines)
+      return false if first_line_has_period?(lines.first.strip)
+
+      remaining_content = lines[1..].join("\n")
+      starts_with_list?(remaining_content.strip)
+    end
+
+    def first_line_is_header?(lines)
+      lines.first&.strip&.end_with?(":") && lines.length > 1
+    end
+
+    def first_line_has_period?(first_line)
+      first_line.count(".").positive?
+    end
+
+    def remove_see_content(notes)
+      notes.each do |note|
+        note.content = note.content.gsub(SEE_TAIL_PATTERN, "")
+      end
+    end
+
+    def remove_redundant_note(notes)
+      notes.reject do |note|
+        note.content.match?(REDUNDANT_NOTE_REGEX) &&
+          !note.content.include?("\n")
+      end
+    end
+
+    def remove_invalid_references(notes)
+      notes.reject do |note|
+        note.content.include?(IMAGE_REF_SUBSTRING) ||
+          note.content.match?(DOUBLE_ANGLE_PATTERN)
+      end
+    end
+
+    def compare_with_definitions(notes, definitions)
+      return [] if notes&.first&.content == definitions&.first&.content
+
+      notes
+    end
+
+    def convert_xref(content)
+      content
+        .gsub(XREF_WITHOUT_DISPLAY) do |_match|
+          full_ref = Regexp.last_match(1)
+          display = full_ref.split(".").last
+          xref_to_mention.call(full_ref, display)
+        end
+        .gsub(XREF_WITH_DISPLAY) do |_match|
+          full_ref = Regexp.last_match(1)
+          display = Regexp.last_match(2)
+          xref_to_mention.call(full_ref, display)
+        end
+    end
+  end
+end

data/lib/suma/processor.rb

@@ -4,15 +4,21 @@ require "metanorma"
 
 module Suma
   class Processor
+    # Emitted collection manifest that both the normal and staged builds render.
+    COLLECTION_OUTPUT_PATH = "collection-output.yaml"
+
     attr_reader :metanorma_yaml_path, :output_directory, :schemas_all_path,
                 :compile_flag
 
     def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
-                   output_directory: "_site")
+                   output_directory: "_site", staged: false)
       @metanorma_yaml_path = metanorma_yaml_path
       @schemas_all_path = schemas_all_path
       @compile_flag = compile
       @output_directory = output_directory
+      # Opt-in memory-bounded staged build (metanorma/suma#94); the default
+      # single-process build below is unchanged when false.
+      @staged = staged
     end
 
     def run
@@ -64,11 +70,21 @@ module Suma
         collection_config, output_directory
       )
 
-      metanorma_collection.render(collection_opts)
+      if @staged
+        # build_collection has written the emitted manifest; stage each member in
+        # its own process, then reinflate. Bounds peak memory to a single member.
+        StagedCollectionBuilder.new(
+          collection_config_path: COLLECTION_OUTPUT_PATH,
+          output_directory: output_directory,
+          coverpage: collection_opts[:coverpage],
+        ).build
+      else
+        metanorma_collection.render(collection_opts)
+      end
     end
 
     def build_collection(collection_config, output_directory)
-      new_collection_config_path = "collection-output.yaml"
+      new_collection_config_path = COLLECTION_OUTPUT_PATH
       ManifestTraverser.new(collection_config.manifest).remove_schemas_only_sources
       collection_config.to_file(new_collection_config_path)
 

data/lib/suma/schema_comparer.rb

@@ -91,7 +91,7 @@ module Suma
         existing_schema = Expressir::Changes::SchemaChange.from_file(output_path)
       end
 
-      converter = EengineConverter.new(xml_path, schema_name)
+      converter = EengineConverter.new(schema_name, File.read(xml_path))
       change_schema = converter.convert(
         version: options[:version],
         existing_change_schema: existing_schema,