expressir 2.3.0 → 2.3.2

data/lib/expressir/express/remark_attacher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 module Expressir
   module Express
     # Handles attaching remarks (comments) to model elements after parsing.
@@ -13,6 +11,17 @@ module Expressir
     # 2. Proximity-based matching for simple tags
     # 3. NOT creating spurious schema-level items for ambiguous tags
     class RemarkAttacher
+      # Child collection attributes to walk when traversing model elements.
+      # These are the named collections on model elements (e.g., entity.attributes,
+      # schema.entities, function.parameters). Extracted to a constant to avoid
+      # duplication between calculate_children_end_line and collect_children.
+      CHILD_COLLECTION_ATTRIBUTES = %i[
+        schemas types entities functions procedures rules constants
+        attributes derived_attributes inverse_attributes
+        where_rules unique_rules informal_propositions
+        parameters variables statements items remark_items
+      ].freeze
+
       def initialize(source)
         @source = source
         @attached_spans = Set.new
@@ -25,8 +34,22 @@ module Expressir
       def attach(model)
         @model = model
         remarks = extract_all_remarks
-        attach_tagged_remarks(model, remarks)
-        attach_untagged_remarks(model, remarks)
+
+        # Build nodes_with_positions ONCE for both tagged and untagged remark passes.
+        # This avoids double tree walk (381K nodes × 2 = 762K visits) which was
+        # the largest memory overhead in remark attachment (~430MB for large files).
+        nodes_with_positions = build_sorted_nodes_with_positions(model)
+
+        attach_tagged_remarks(model, remarks, nodes_with_positions)
+        attach_untagged_remarks(remarks, nodes_with_positions)
+
+        # Free expensive data structures after attachment is complete.
+        # These are only needed during the attach process.
+        @source = nil
+        @source_lines = nil
+        @scope_map = nil
+        @line_cache = nil
+
         model
       end
 
@@ -124,9 +147,7 @@ module Expressir
         @line_cache[position] ||= @source.byteslice(0...position).count("\n") + 1
       end
 
-      alias get_line_number_from_offset get_line_number
-
-      def attach_tagged_remarks(model, remarks)
+      def attach_tagged_remarks(model, remarks, nodes_with_positions)
         tagged = remarks.select { |r| r[:tag] }
         return if tagged.empty?
 
@@ -136,12 +157,6 @@ module Expressir
         # This is the key optimization that makes scope lookup O(1) per remark
         @scope_map ||= build_scope_map
 
-        # Collect nodes with positions for position-based fallback
-        nodes_with_positions = []
-        collect_nodes_with_positions(model, nodes_with_positions)
-        # Use stable sort to ensure deterministic ordering across Ruby versions
-        nodes_with_positions.sort_by!.with_index { |n, i| [n[:position] || Float::INFINITY, i] }
-
         tagged.sort_by { |r| r[:position] }.each do |remark|
           next if @attached_spans.include?(remark[:position])
 
@@ -242,7 +257,8 @@ module Expressir
               end
               # Fall back to schema prefix
               if target.nil?
-                target = create_implicit_remark_item(model, tag, get_schema_ids(model))
+                target = create_implicit_remark_item(model, tag,
+                                                     get_schema_ids(model))
               end
             end
 
@@ -280,6 +296,7 @@ module Expressir
 
         nil
       end
+
       # Done once per RemarkAttacher instance (O(file_lines)).
       # Each find_containing_scope call then becomes O(1).
       def build_scope_map
@@ -557,102 +574,6 @@ module Expressir
         find_containing_scope_position(remark_line, nodes_with_positions)
       end
 
-      def find_scope_by_text_search(remark_line)
-        lines = source_lines
-        return nil if remark_line < 1 || remark_line > lines.length
-
-        # Track nested scopes by searching backwards from remark_line
-        scope_stack = []
-
-        lines.each_with_index do |line, idx|
-          line_num = idx + 1
-          break if line_num > remark_line
-
-          # Check for START keywords first
-          if line =~ /^\s*SCHEMA\s+(\w+)/i
-            scope_stack << { type: :schema, name: $1, line: line_num }
-          end
-
-          if line =~ /^\s*FUNCTION\s+(\w+)/i
-            scope_stack << { type: :function, name: $1, line: line_num }
-          end
-
-          if line =~ /^\s*PROCEDURE\s+(\w+)/i
-            scope_stack << { type: :procedure, name: $1, line: line_num }
-          end
-
-          if line =~ /^\s*RULE\s+(\w+)/i
-            scope_stack << { type: :rule, name: $1, line: line_num }
-          end
-
-          if line =~ /^\s*ENTITY\s+(\w+)/i
-            scope_stack << { type: :entity, name: $1, line: line_num }
-          end
-
-          if line =~ /^\s*TYPE\s+(\w+)/i
-            scope_stack << { type: :type, name: $1, line: line_num }
-          end
-
-          # Then check for END keywords (to handle inline closures on same line)
-          if (line =~ /END_TYPE/i) && (scope_stack.last&.dig(:type) == :type)
-            scope_stack.pop
-          end
-          if (line =~ /END_FUNCTION/i) && (scope_stack.last&.dig(:type) == :function)
-            scope_stack.pop
-          end
-          if (line =~ /END_PROCEDURE/i) && (scope_stack.last&.dig(:type) == :procedure)
-            scope_stack.pop
-          end
-          if (line =~ /END_RULE/i) && (scope_stack.last&.dig(:type) == :rule)
-            scope_stack.pop
-          end
-          if (line =~ /END_ENTITY/i) && (scope_stack.last&.dig(:type) == :entity)
-            scope_stack.pop
-          end
-          if (line =~ /END_SCHEMA/i) && (scope_stack.last&.dig(:type) == :schema)
-            scope_stack.pop
-          end
-        end
-
-        # Find the innermost scope and get the corresponding model node
-        return nil if scope_stack.empty?
-
-        innermost = scope_stack.last
-        find_scope_node(innermost[:type], innermost[:name])
-      end
-
-      def find_scope_node(type, name)
-        return nil unless @model && name
-
-        @model.schemas.each do |schema|
-          # Check schema itself
-          if type == :schema && schema.id == name
-            return schema
-          end
-
-          # Check schema-level declarations
-          case type
-          when :function
-            found = schema.functions&.find { |f| f.id == name }
-            return found if found
-          when :procedure
-            found = schema.procedures&.find { |p| p.id == name }
-            return found if found
-          when :rule
-            found = schema.rules&.find { |r| r.id == name }
-            return found if found
-          when :entity
-            found = schema.entities&.find { |e| e.id == name }
-            return found if found
-          when :type
-            found = schema.types&.find { |t| t.id == name }
-            return found if found
-          end
-        end
-
-        nil
-      end
-
       def build_scope_path(node)
         return nil unless node
 
@@ -673,36 +594,6 @@ module Expressir
         parts.empty? ? nil : parts.join(".")
       end
 
-      def find_containing_scope_for_ip(remark_line, nodes_with_positions)
-        # First try scope map (O(1))
-        scope = find_containing_scope_by_name(remark_line)
-        if scope && supports_informal_propositions?(scope)
-          return scope
-        end
-
-        # Fallback to position-based detection
-        containing_nodes = nodes_with_positions.select do |n|
-          n[:line] && n[:end_line] && remark_line >= n[:line] && remark_line <= n[:end_line] &&
-            !repository?(n[:node]) && !cache?(n[:node])
-        end
-
-        # Find the innermost node that supports informal propositions
-        if containing_nodes.any?
-          containing_nodes.reverse_each do |n|
-            node = n[:node]
-            if supports_informal_propositions?(node)
-              return node
-            end
-
-            # Fallback to schema
-            return node if node.is_a?(Model::Declarations::Schema)
-          end
-        end
-
-        # Fallback: search for containing entity/type/rule by source text
-        find_scope_by_source_text(remark_line)
-      end
-
       def find_scope_by_source_text(remark_line)
         # Search backwards from remark_line for containing scope
         lines = source_lines
@@ -908,15 +799,10 @@ module Expressir
         item
       end
 
-      def attach_untagged_remarks(model, remarks)
+      def attach_untagged_remarks(remarks, nodes_with_positions)
         untagged = remarks.reject { |r| r[:tag] }
         return unless untagged.any?
 
-        nodes_with_positions = []
-        collect_nodes_with_positions(model, nodes_with_positions)
-        # Use stable sort to preserve original order for equal keys
-        nodes_with_positions.sort_by!.with_index { |n, i| [n[:position] || Float::INFINITY, i] }
-
         untagged.each do |remark|
           next if @attached_spans.include?(remark[:position])
 
@@ -1021,21 +907,41 @@ module Expressir
           # The parser always provides this via Slice#offset
           if node.source_offset
             pos = node.source_offset
-            line = get_line_number(pos)
-            source_end_line = get_line_number(pos + node.source.length)
-
-            # For container nodes, use the maximum end_line from children
-            # This is needed because source.length only covers the declaration, not the body
-            children_end_line = calculate_children_end_line(node)
-            end_line = [source_end_line,
-                        children_end_line].compact.max || source_end_line
-
-            result << {
-              node: node,
-              position: pos,
-              line: line,
-              end_line: end_line,
-            }
+            # Validate offset: native parser returns 0 for leaf nodes (WhereRule)
+            # where it can't determine the actual position. These have short
+            # expression-like source ("TRUE;") that doesn't appear at file start.
+            # Container nodes (Schema, Entity, Type) have declaration-like source
+            # that either starts at position 0 legitimately or is clearly valid.
+            valid = pos.positive?
+            if !valid && pos.zero? && node.source
+              src = node.source.to_s
+              # Accept position=0 if source is a declaration keyword line
+              valid = src.start_with?("SCHEMA", "ENTITY", "TYPE", "FUNCTION",
+                                     "PROCEDURE", "RULE", "CONSTANT", "VARIABLE",
+                                     "USE", "REFERENCE", "END_SCHEMA", "END_ENTITY",
+                                     "END_TYPE", "END_FUNCTION", "END_PROCEDURE",
+                                     "END_RULE", "END_CONSTANT", "END_VARIABLE")
+            end
+            if valid
+              line = get_line_number(pos)
+              source_end_line = get_line_number(pos + node.source.length)
+
+              # For container nodes, use the maximum end_line from children
+              # This is needed because source.length only covers the declaration, not the body
+              children_end_line = calculate_children_end_line(node)
+              end_line = [source_end_line,
+                          children_end_line].compact.max || source_end_line
+
+              result << {
+                node: node,
+                position: pos,
+                line: line,
+                end_line: end_line,
+              }
+            else
+              # Invalid offset — treat as unknown position
+              result << { node: node, position: nil, line: nil, end_line: nil }
+            end
           else
             # No source_offset available - should not happen if parser provides Slice
             result << { node: node, position: nil, line: nil, end_line: nil }
@@ -1063,10 +969,7 @@ module Expressir
         end
 
         # Check specific child collections
-        %i[schemas types entities functions procedures rules constants
-           attributes derived_attributes inverse_attributes
-           where_rules unique_rules informal_propositions
-           parameters variables statements items remark_items].each do |attr|
+        CHILD_COLLECTION_ATTRIBUTES.each do |attr|
           collection = safe_get_collection(node, attr)
           collection&.each do |child|
             if child.is_a?(Model::ModelElement) && child.source_offset && child.source
@@ -1085,10 +988,7 @@ module Expressir
           collect_nodes_with_positions(child, result, visited)
         end
 
-        %i[schemas types entities functions procedures rules constants
-           attributes derived_attributes inverse_attributes
-           where_rules unique_rules informal_propositions
-           parameters variables statements items remark_items].each do |attr|
+        CHILD_COLLECTION_ATTRIBUTES.each do |attr|
           collection = safe_get_collection(node, attr)
           collection&.each do |item|
             collect_nodes_with_positions(item, result, visited)
@@ -1096,6 +996,16 @@ module Expressir
         end
       end
 
+      # Build sorted nodes_with_positions ONCE for both tagged and untagged remark passes.
+      # This merges the two separate tree walks into one, cutting node visits in half.
+      def build_sorted_nodes_with_positions(model)
+        nodes_with_positions = []
+        collect_nodes_with_positions(model, nodes_with_positions)
+        # Stable sort: nil positions last, ties broken by insertion order
+        nodes_with_positions.sort_by!.with_index { |n, i| [n[:position] || Float::INFINITY, i] }
+        nodes_with_positions
+      end
+
       def find_nearest_node(remark, nodes)
         remark_line = remark[:line]
 

data/lib/expressir/express/transformer/remark_handling.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "parsanol"
-require "set"
 
 module Expressir
   module Express

data/lib/expressir/model/declarations/schema.rb

@@ -21,7 +21,6 @@ module Expressir
         attribute :selected, :boolean, default: false
         attribute :formatted, :string
         attribute :file_basename, :string
-        attribute :full_source, :string
 
         key_value do
           map "_class", to: :_class, render_default: true

data/lib/expressir/model/exp_file.rb

@@ -14,7 +14,8 @@ module Expressir
       include ScopeContainer
 
       attribute :path, :string
-      attribute :schemas, Declarations::Schema, collection: true, initialize_empty: true
+      attribute :schemas, Declarations::Schema, collection: true,
+                                                initialize_empty: true
       attribute :_class, :string, default: -> { self.class.name }
 
       key_value do

data/lib/expressir/model/model_element.rb

@@ -208,7 +208,7 @@ module Expressir
       def remark_infos
         return [] if untagged_remarks.nil?
 
-        untagged_remarks.select { |r| r.is_a?(RemarkInfo) }
+        untagged_remarks.grep(RemarkInfo)
       end
 
       private

data/lib/expressir/model/repository.rb

@@ -32,8 +32,8 @@ module Expressir
         map "files", to: :files
       end
 
-      def initialize(*args, base_dir: nil, schemas: nil, **kwargs)
-        super(*args, **kwargs)
+      def initialize(*, base_dir: nil, schemas: nil, **)
+        super(*, **)
         @base_dir = base_dir
         @entity_index = nil
         @type_index = nil
@@ -41,9 +41,7 @@ module Expressir
         @_schemas = []
 
         # Support direct schemas initialization
-        if schemas
-          schemas.each { |schema| @_schemas << schema }
-        end
+        schemas&.each { |schema| @_schemas << schema }
       end
 
       # Get all schemas (from both files and direct storage)
@@ -63,6 +61,7 @@ module Expressir
       # @param schema [Declarations::Schema] Schema to add
       def add_schema(schema)
         return unless schema
+
         @_schemas << schema
       end
 
@@ -342,16 +341,16 @@ module Expressir
       # @param format [Symbol] Output format
       # @yield [item] Block to convert item to hash
       # @return [Array, String] Formatted output
-      def format_output(items, format, &block)
+      def format_output(items, format, &)
         case format
         when :hash
-          items.map(&block)
+          items.map(&)
         when :json
           require "json"
-          items.map(&block).to_json
+          items.map(&).to_json
         when :yaml
           require "yaml"
-          items.map(&block).to_yaml
+          items.map(&).to_yaml
         else
           items
         end

data/lib/expressir/model/search_engine.rb

@@ -84,10 +84,10 @@ module Expressir
       # @param max_depth [Integer] Maximum path depth (1=schema, 2=entity, 3=attribute)
       # @param options [Hash] Additional search options
       # @return [Array<Hash>] Filtered results
-      def search_with_depth(pattern:, max_depth:, **options)
+      def search_with_depth(pattern:, max_depth:, **)
         return [] if max_depth <= 0
 
-        results = search(pattern: pattern, **options)
+        results = search(pattern: pattern, **)
         filter_by_depth(results, max_depth)
       end
 
@@ -97,8 +97,8 @@ module Expressir
       # @param boost_prefix [Integer] Score boost for prefix matches
       # @param options [Hash] Additional search options
       # @return [Array<Hash>] Ranked results with :relevance_score
-      def search_ranked(pattern:, boost_exact: 10, boost_prefix: 5, **options)
-        results = search(pattern: pattern, **options)
+      def search_ranked(pattern:, boost_exact: 10, boost_prefix: 5, **)
+        results = search(pattern: pattern, **)
         rank_results(results, pattern, boost_exact, boost_prefix)
       end
 
@@ -108,8 +108,8 @@ module Expressir
       # @param ranked [Boolean] Enable relevance ranking
       # @param options [Hash] Additional filters and search options
       # @return [Array<Hash>] Filtered and optionally ranked results
-      def search_advanced(pattern:, max_depth: nil, ranked: false, **options)
-        results = search(pattern: pattern, **options)
+      def search_advanced(pattern:, max_depth: nil, ranked: false, **)
+        results = search(pattern: pattern, **)
 
         results = filter_by_depth(results, max_depth) if max_depth
         results = rank_results(results, pattern, 10, 5) if ranked
@@ -485,6 +485,7 @@ module Expressir
         current = element
         while current
           return current if current.is_a?(Declarations::Schema)
+
           current = current.parent
         end
       rescue StandardError

data/lib/expressir/package/builder.rb

@@ -152,7 +152,9 @@ module Expressir
 
         # Get schemas that are not already in files
         file_schema_ids = normalized.files.flat_map(&:schemas).compact.map(&:id)
-        direct_schemas = repository.schemas.reject { |s| file_schema_ids.include?(s.id) }
+        direct_schemas = repository.schemas.reject do |s|
+          file_schema_ids.include?(s.id)
+        end
 
         # Create an ExpFile for direct schemas
         if direct_schemas.any?

data/lib/expressir/package/metadata.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-
 module Expressir
   module Package
     # Package metadata with serialization support

data/lib/expressir/schema_manifest.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-
 module Expressir
   class SchemaManifest < Lutaml::Model::Serializable
     attribute :schemas, "Expressir::SchemaManifestEntry",

data/lib/expressir/schema_manifest_entry.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-
 module Expressir
   class SchemaManifestEntry < Lutaml::Model::Serializable
     attribute :id, :string

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "2.3.0".freeze
+  VERSION = "2.3.2".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: expressir
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-03-24 00:00:00.000000000 Z
+date: 2026-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: paint
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,39 +123,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: parsanol
+  name: paint
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.3.7
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.7
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.3.7
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.7
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: parsanol
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.9
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.9
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.9
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement