expressir 2.2.1 → 2.3.1

data/lib/expressir/express/remark_attacher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 module Expressir
   module Express
     # Handles attaching remarks (comments) to model elements after parsing.
@@ -13,89 +11,13 @@ module Expressir
     # 2. Proximity-based matching for simple tags
     # 3. NOT creating spurious schema-level items for ambiguous tags
     class RemarkAttacher
-      # Types that support informal propositions
-      INFORMAL_PROPOSITION_TYPES = [
-        Model::Declarations::Entity,
-        Model::Declarations::Rule,
-        Model::Declarations::Type,
-        Model::Declarations::InformalPropositionRule,
-      ].freeze
-
-      # Types that support remark items (have remark_items attribute)
-      # These are types where we can create RemarkItem children
-      REMARK_ITEM_TYPES = [
-        Model::Declarations::Schema,
-        Model::Declarations::Entity,
-        Model::Declarations::Type,
-        Model::Declarations::Rule,
-        Model::Declarations::Function,
-        Model::Declarations::Procedure,
-        Model::Declarations::InformalPropositionRule,
-        Model::Declarations::WhereRule,
-        Model::Declarations::UniqueRule,
-        # Attribute types (all include Identifier which provides remark_items)
-        Model::Declarations::Attribute,
-        Model::Declarations::DerivedAttribute,
-        Model::Declarations::InverseAttribute,
-      ].freeze
-
-      # Types that support where rules
-      WHERE_RULE_TYPES = [
-        Model::Declarations::Entity,
-        Model::Declarations::Type,
-        Model::Declarations::Rule,
-        Model::Declarations::Function,
-        Model::Declarations::Procedure,
-      ].freeze
-
-      # Scope container types (can contain other declarations)
-      SCOPE_CONTAINER_TYPES = [
-        Model::Declarations::Schema,
-        Model::Declarations::Function,
-        Model::Declarations::Procedure,
-        Model::Declarations::Rule,
-        Model::Declarations::Entity,
-        Model::Declarations::Type,
-      ].freeze
-
-      # Types that support remarks (have Identifier module or define remarks directly)
-      REMARKS_SUPPORT_TYPES = [
-        # Declaration types with Identifier module
-        Model::Declarations::Schema,
-        Model::Declarations::Entity,
-        Model::Declarations::Type,
-        Model::Declarations::Function,
-        Model::Declarations::Procedure,
-        Model::Declarations::Rule,
-        Model::Declarations::Constant,
-        Model::Declarations::Attribute,
-        Model::Declarations::InverseAttribute,
-        Model::Declarations::DerivedAttribute,
-        Model::Declarations::WhereRule,
-        Model::Declarations::UniqueRule,
-        Model::Declarations::InformalPropositionRule,
-        Model::Declarations::SubtypeConstraint,
-        Model::Declarations::Parameter,
-        Model::Declarations::Variable,
-        # Statement types with Identifier module
-        Model::Statements::Alias,
-        Model::Statements::Repeat,
-        # Expression types with Identifier module
-        Model::Expressions::QueryExpression,
-        # Data types with Identifier module
-        Model::DataTypes::Aggregate,
-        Model::DataTypes::EnumerationItem,
-        Model::DataTypes::Generic,
-        Model::DataTypes::GenericEntity,
-        # Types with remarks attribute defined directly (not via Identifier)
-        Model::Declarations::RemarkItem,
-      ].freeze
-
       def initialize(source)
         @source = source
         @attached_spans = Set.new
         @line_cache = {}
         @model = nil
+        @source_lines = nil # cached @source.lines
+        @scope_map = nil # cached scope at each line number
       end
 
       def attach(model)
@@ -103,6 +25,15 @@ module Expressir
         remarks = extract_all_remarks
         attach_tagged_remarks(model, remarks)
         attach_untagged_remarks(model, remarks)
+
+        # Free expensive data structures after attachment is complete.
+        # These are only needed during the attach process.
+        @source = nil
+        @source_lines = nil
+        @scope_map = nil
+        @line_cache = nil
+        @remarks_cache = nil
+
         model
       end
 
@@ -190,51 +121,63 @@ module Expressir
         end
       end
 
+      def source_lines
+        @source_lines ||= @source.lines
+      end
+
       def get_line_number(position)
         return 1 if position.nil? || position.zero?
 
         @line_cache[position] ||= @source.byteslice(0...position).count("\n") + 1
       end
 
+      alias get_line_number_from_offset get_line_number
+
       def attach_tagged_remarks(model, remarks)
-        schema_ids = repository?(model) ? model.schemas.filter_map(&:id) : []
+        tagged = remarks.select { |r| r[:tag] }
+        return if tagged.empty?
+
+        @model = model
+
+        # Build scope map ONCE: O(file_lines) scan instead of O(n*file_lines) for n remarks
+        # This is the key optimization that makes scope lookup O(1) per remark
+        @scope_map ||= build_scope_map
 
-        # Collect nodes with positions for finding containing scopes
+        # Collect nodes with positions for position-based fallback
         nodes_with_positions = []
         collect_nodes_with_positions(model, nodes_with_positions)
         # Use stable sort to ensure deterministic ordering across Ruby versions
-        # When positions are equal, preserve original order using index as tie-breaker
         nodes_with_positions.sort_by!.with_index { |n, i| [n[:position] || Float::INFINITY, i] }
 
-        remarks.select do |r|
-          r[:tag]
-        end.sort_by { |r| r[:position] }.each do |remark|
+        tagged.sort_by { |r| r[:position] }.each do |remark|
           next if @attached_spans.include?(remark[:position])
 
           tag = remark[:tag]
           target = nil
 
+          # Find containing scope using pre-computed scope map (O(1))
+          # Falls back to position-based lookup if scope map doesn't have the line
+          containing_scope = find_containing_scope_by_name(remark[:line])
+          containing_scope ||= find_containing_scope_position(remark[:line],
+                                                              nodes_with_positions)
+
           # Check if this is an informal proposition tag (IP\d+)
           if tag.match?(/^IP\d+$/)
-            # Find the containing scope (entity, type, rule) that supports informal propositions
-            target = find_containing_scope_for_ip(remark[:line],
-                                                  nodes_with_positions)
-            if target
-              # Create or find the informal proposition
-              target = create_or_find_informal_proposition(target, tag)
+            scope = containing_scope
+            if scope.nil?
+              scope = find_scope_by_source_text(remark[:line])
+            end
+            if scope && supports_informal_propositions?(scope)
+              target = create_or_find_informal_proposition(scope, tag)
             end
           end
 
           # Standard path-based lookup
           if target.nil?
-            # Find containing scope for scope-aware path resolution
-            containing_scope = find_containing_scope(remark[:line],
-                                                     nodes_with_positions)
-
             # Handle prefixed tags like wr:WR1, ip:IP1, ur:UR1
             if tag.include?(":") && !tag.include?(".")
               target = handle_prefixed_tag(tag, containing_scope, model,
-                                           schema_ids)
+                                           get_schema_ids(model))
             end
 
             # Strategy 1: Try exact path lookup
@@ -255,6 +198,7 @@ module Expressir
 
               # Then try schema prefix
               if target.nil?
+                schema_ids = get_schema_ids(model)
                 schema_ids.each do |schema_id|
                   target = find_by_exact_path(model, "#{schema_id}.#{tag}")
                   break if target
@@ -275,8 +219,8 @@ module Expressir
                 end
 
                 # Only fall back to schema prefix if NOT inside a function/rule/procedure
-                # This prevents remarks inside scopes from attaching to schema-level items
                 if target.nil? && !function_rule_procedure?(containing_scope)
+                  schema_ids = get_schema_ids(model)
                   schema_ids.each do |schema_id|
                     target = find_by_exact_path(model, "#{schema_id}.#{tag}")
                     break if target
@@ -284,6 +228,7 @@ module Expressir
                 end
               else
                 # No containing scope, try with schema prefix
+                schema_ids = get_schema_ids(model)
                 schema_ids.each do |schema_id|
                   target = find_by_exact_path(model, "#{schema_id}.#{tag}")
                   break if target
@@ -299,19 +244,23 @@ module Expressir
                 if scope_path
                   full_path = "#{scope_path}.#{tag}"
                   target = create_implicit_remark_item(model, full_path,
-                                                       schema_ids)
+                                                       get_schema_ids(model))
                 end
               end
               # Fall back to schema prefix
               if target.nil?
-                target = create_implicit_remark_item(model, tag, schema_ids)
+                target = create_implicit_remark_item(model, tag,
+                                                     get_schema_ids(model))
               end
             end
 
             # Strategy 4: For simple tags at schema level, create implicit item
-            if target.nil? && !tag.include?(".") && schema_ids.any?
-              target = create_implicit_remark_item_at_schema(model, tag,
-                                                             schema_ids.first)
+            if target.nil? && !tag.include?(".")
+              schema_ids = get_schema_ids(model)
+              if schema_ids.any?
+                target = create_implicit_remark_item_at_schema(model, tag,
+                                                               schema_ids.first)
+              end
             end
           end
 
@@ -323,6 +272,113 @@ module Expressir
         end
       end
 
+      # Position-based fallback for finding containing scope.
+      # Used when scope map lookup returns nil (e.g., for remarks at lines
+      # outside any declared scope's end_line, or for non-scope-containers).
+      def find_containing_scope_position(remark_line, nodes_with_positions)
+        containing_nodes = nodes_with_positions.select do |n|
+          n[:line] && n[:end_line] && remark_line >= n[:line] && remark_line <= n[:end_line] &&
+            !repository?(n[:node]) && !cache?(n[:node])
+        end
+
+        containing_nodes.reverse_each do |n|
+          node = n[:node]
+          return node if node.is_a?(Model::ScopeContainer)
+        end
+
+        nil
+      end
+
+      # Done once per RemarkAttacher instance (O(file_lines)).
+      # Each find_containing_scope call then becomes O(1).
+      def build_scope_map
+        lines = source_lines
+        scope_map = {}
+        return scope_map if lines.empty?
+
+        # Track nested scopes by scanning all lines once
+        scope_stack = [] # array of {type:, name:, line:}
+
+        lines.each_with_index do |line, idx|
+          line_num = idx + 1
+
+          # Check for START keywords first
+          if line =~ /^\s*SCHEMA\s+(\w+)/i
+            scope_stack << { type: :schema, name: $1, line: line_num }
+          end
+
+          if line =~ /^\s*FUNCTION\s+(\w+)/i
+            scope_stack << { type: :function, name: $1, line: line_num }
+          end
+
+          if line =~ /^\s*PROCEDURE\s+(\w+)/i
+            scope_stack << { type: :procedure, name: $1, line: line_num }
+          end
+
+          if line =~ /^\s*RULE\s+(\w+)/i
+            scope_stack << { type: :rule, name: $1, line: line_num }
+          end
+
+          if line =~ /^\s*ENTITY\s+(\w+)/i
+            scope_stack << { type: :entity, name: $1, line: line_num }
+          end
+
+          if line =~ /^\s*TYPE\s+(\w+)/i
+            scope_stack << { type: :type, name: $1, line: line_num }
+          end
+
+          # Check for END keywords (inline closures on same line handled here)
+          if (line =~ /END_TYPE/i) && (scope_stack.last&.dig(:type) == :type)
+            scope_stack.pop
+          end
+          if (line =~ /END_FUNCTION/i) && (scope_stack.last&.dig(:type) == :function)
+            scope_stack.pop
+          end
+          if (line =~ /END_PROCEDURE/i) && (scope_stack.last&.dig(:type) == :procedure)
+            scope_stack.pop
+          end
+          if (line =~ /END_RULE/i) && (scope_stack.last&.dig(:type) == :rule)
+            scope_stack.pop
+          end
+          if (line =~ /END_ENTITY/i) && (scope_stack.last&.dig(:type) == :entity)
+            scope_stack.pop
+          end
+          if (line =~ /END_SCHEMA/i) && (scope_stack.last&.dig(:type) == :schema)
+            scope_stack.pop
+          end
+
+          # Record the innermost scope for this line
+          scope_map[line_num] = scope_stack.last&.dig(:name)
+        end
+
+        scope_map
+      end
+
+      # O(1) scope lookup using pre-computed scope map
+      def find_containing_scope_by_name(remark_line)
+        return nil unless @scope_map
+
+        scope_name = @scope_map[remark_line]
+        return nil unless scope_name
+
+        # Find the model node for this scope
+        return nil unless @model
+
+        @model.schemas.each do |schema|
+          return schema if schema.id == scope_name
+
+          %i[functions procedures rules entities types].each do |decl_type|
+            collection = schema.send(decl_type)
+            next unless collection.is_a?(Array)
+
+            found = collection.find { |n| n.id == scope_name }
+            return found if found
+          end
+        end
+
+        nil
+      end
+
       def find_node_in_scope(scope, tag)
         return nil unless scope
 
@@ -465,7 +521,7 @@ module Expressir
         return nil unless where_rules&.any?
 
         # Search source text for WHERE clause containing this remark
-        lines = @source.lines
+        lines = source_lines
 
         where_rules.each do |wr|
           next unless wr.id
@@ -502,30 +558,16 @@ module Expressir
       end
 
       def find_containing_scope(remark_line, nodes_with_positions)
-        # First try text-based detection (more reliable when source tracking is broken)
-        text_based_scope = find_scope_by_text_search(remark_line)
-        return text_based_scope if text_based_scope
+        # First try scope map (O(1) once built)
+        scope = find_containing_scope_by_name(remark_line)
+        return scope if scope
 
         # Fallback to position-based detection
-        # Exclude Repository and Cache as they are not semantic scopes
-        containing_nodes = nodes_with_positions.select do |n|
-          n[:line] && n[:end_line] && remark_line >= n[:line] && remark_line <= n[:end_line] &&
-            !repository?(n[:node]) && !cache?(n[:node])
-        end
-
-        # Return the innermost scope container (function, procedure, rule, entity, type)
-        containing_nodes.reverse_each do |n|
-          node = n[:node]
-          SCOPE_CONTAINER_TYPES.each do |scope_class|
-            return node if node.is_a?(scope_class)
-          end
-        end
-
-        nil
+        find_containing_scope_position(remark_line, nodes_with_positions)
       end
 
       def find_scope_by_text_search(remark_line)
-        lines = @source.lines
+        lines = source_lines
         return nil if remark_line < 1 || remark_line > lines.length
 
         # Track nested scopes by searching backwards from remark_line
@@ -641,23 +683,19 @@ module Expressir
       end
 
       def find_containing_scope_for_ip(remark_line, nodes_with_positions)
-        # First try text-based detection (more reliable when source tracking is broken)
-        # This handles cases where node end_line doesn't include trailing remarks
-        text_based_scope = find_scope_by_text_search(remark_line)
-        if text_based_scope && supports_informal_propositions?(text_based_scope)
-          return text_based_scope
+        # First try scope map (O(1))
+        scope = find_containing_scope_by_name(remark_line)
+        if scope && supports_informal_propositions?(scope)
+          return scope
         end
 
         # Fallback to position-based detection
-        # Find nodes that contain this remark line
-        # Exclude Repository and Cache as they are not semantic scopes
         containing_nodes = nodes_with_positions.select do |n|
           n[:line] && n[:end_line] && remark_line >= n[:line] && remark_line <= n[:end_line] &&
             !repository?(n[:node]) && !cache?(n[:node])
         end
 
         # Find the innermost node that supports informal propositions
-        # Priority: Entity, Rule, Type, Schema
         if containing_nodes.any?
           containing_nodes.reverse_each do |n|
             node = n[:node]
@@ -676,7 +714,7 @@ module Expressir
 
       def find_scope_by_source_text(remark_line)
         # Search backwards from remark_line for containing scope
-        lines = @source.lines
+        lines = source_lines
 
         # Find the entity/type/rule that contains this line
         entity_start = nil
@@ -756,7 +794,10 @@ module Expressir
       end
 
       def find_by_exact_path(model, path)
-        return nil unless path && repository?(model)
+        return nil unless path
+
+        # Only Repository and ExpFile support path-based find
+        return nil unless repository?(model) || exp_file?(model)
 
         # Try original path
         result = safe_find(model, path)
@@ -768,7 +809,8 @@ module Expressir
       end
 
       def create_implicit_remark_item_at_schema(model, item_id, schema_id)
-        return nil unless repository?(model)
+        # Only Repository and ExpFile support schema lookup
+        return nil unless repository?(model) || exp_file?(model)
 
         schema = safe_find(model, schema_id)
         return nil unless schema.is_a?(Model::Declarations::Schema)
@@ -789,7 +831,7 @@ module Expressir
       end
 
       def create_implicit_remark_item(model, path, schema_ids = [])
-        return nil unless repository?(model)
+        return nil unless repository?(model) || exp_file?(model)
 
         # Normalize path (handle "ip:IP1" format)
         clean_path = normalize_path(path)
@@ -913,7 +955,7 @@ module Expressir
       end
 
       def get_line_content(line_num)
-        lines = @source.lines
+        lines = source_lines
         return "" if line_num < 1 || line_num > lines.length
 
         lines[line_num - 1]
@@ -952,20 +994,29 @@ module Expressir
 
         # Only add remarks to nodes that support them
         if supports_remarks?(node)
-          node.remarks ||= []
-          node.remarks << text
+          # Always add to remarks attribute (for types that have it)
+          if node.respond_to?(:remarks) && node.respond_to?(:remarks=)
+            node.remarks ||= []
+            node.remarks << text
+          end
 
           if tag.nil?
-            remark_info = Model::RemarkInfo.new(text: text, format: format,
-                                                tag: tag)
+            # Untagged remark: store in untagged_remarks
+            remark_info = Model::RemarkInfo.new(text: text, format: format)
             node.untagged_remarks ||= []
             node.untagged_remarks << remark_info
           end
         end
       end
 
+      # All ModelElement subclasses have untagged_remarks from ModelElement
       def supports_remarks?(obj)
-        REMARKS_SUPPORT_TYPES.any? { |t| obj.is_a?(t) }
+        obj.is_a?(Model::ModelElement)
+      end
+
+      # Types that include HasRemarkItems can have remark_items
+      def supports_remark_items?(obj)
+        obj.is_a?(Model::HasRemarkItems)
       end
 
       def collect_nodes_with_positions(node, result, visited = Set.new)
@@ -979,20 +1030,41 @@ module Expressir
           # The parser always provides this via Slice#offset
           if node.source_offset
             pos = node.source_offset
-            line = get_line_number(pos)
-            source_end_line = get_line_number(pos + node.source.length)
-
-            # For container nodes, use the maximum end_line from children
-            # This is needed because source.length only covers the declaration, not the body
-            children_end_line = calculate_children_end_line(node)
-            end_line = [source_end_line, children_end_line].compact.max || source_end_line
-
-            result << {
-              node: node,
-              position: pos,
-              line: line,
-              end_line: end_line,
-            }
+            # Validate offset: native parser returns 0 for leaf nodes (WhereRule)
+            # where it can't determine the actual position. These have short
+            # expression-like source ("TRUE;") that doesn't appear at file start.
+            # Container nodes (Schema, Entity, Type) have declaration-like source
+            # that either starts at position 0 legitimately or is clearly valid.
+            valid = pos.positive?
+            if !valid && pos.zero? && node.source
+              src = node.source.to_s
+              # Accept position=0 if source is a declaration keyword line
+              valid = src.start_with?("SCHEMA", "ENTITY", "TYPE", "FUNCTION",
+                                     "PROCEDURE", "RULE", "CONSTANT", "VARIABLE",
+                                     "USE", "REFERENCE", "END_SCHEMA", "END_ENTITY",
+                                     "END_TYPE", "END_FUNCTION", "END_PROCEDURE",
+                                     "END_RULE", "END_CONSTANT", "END_VARIABLE")
+            end
+            if valid
+              line = get_line_number(pos)
+              source_end_line = get_line_number(pos + node.source.length)
+
+              # For container nodes, use the maximum end_line from children
+              # This is needed because source.length only covers the declaration, not the body
+              children_end_line = calculate_children_end_line(node)
+              end_line = [source_end_line,
+                          children_end_line].compact.max || source_end_line
+
+              result << {
+                node: node,
+                position: pos,
+                line: line,
+                end_line: end_line,
+              }
+            else
+              # Invalid offset — treat as unknown position
+              result << { node: node, position: nil, line: nil, end_line: nil }
+            end
           else
             # No source_offset available - should not happen if parser provides Slice
             result << { node: node, position: nil, line: nil, end_line: nil }
@@ -1075,13 +1147,23 @@ module Expressir
         # Find the node that CONTAINS this remark line
         # This handles preamble remarks and embedded remarks
         # Exclude Repository and Cache as they are not semantic scopes
+        # But include ExpFile for file-level preamble remarks
         containing = nodes.select do |n|
           n[:line] && n[:end_line] && n[:line] <= remark_line && n[:end_line] >= remark_line &&
             !repository?(n[:node]) && !cache?(n[:node])
         end
 
         if containing.any?
-          # Return the most specific (smallest) containing node
+          # Prefer ExpFile for preamble remarks (before first schema)
+          # Otherwise return the most specific (smallest) containing node
+          exp_file_node = containing.find { |n| exp_file?(n[:node]) }
+          # If this is a preamble remark (before first schema line), use ExpFile
+          if exp_file_node
+            first_schema_line = exp_file_node[:node].schemas&.first&.source_offset
+            if first_schema_line && remark_line < get_line_number(first_schema_line)
+              return exp_file_node[:node]
+            end
+          end
           # Sort by span size and return the smallest
           containing.min_by { |n| n[:end_line] - n[:line] }[:node]
         else
@@ -1096,24 +1178,34 @@ module Expressir
 
       # Type checking helper methods
 
+      def get_schema_ids(model)
+        if repository?(model)
+          model.schemas.filter_map(&:id)
+        elsif exp_file?(model)
+          model.schemas.filter_map(&:id)
+        else
+          []
+        end
+      end
+
       def repository?(obj)
         obj.is_a?(Model::Repository)
       end
 
+      def exp_file?(obj)
+        obj.is_a?(Model::ExpFile)
+      end
+
       def cache?(obj)
         obj.is_a?(Model::Cache)
       end
 
       def supports_informal_propositions?(obj)
-        INFORMAL_PROPOSITION_TYPES.any? { |t| obj.is_a?(t) }
-      end
-
-      def supports_remark_items?(obj)
-        REMARK_ITEM_TYPES.any? { |t| obj.is_a?(t) }
+        obj.is_a?(Model::HasInformalPropositions)
       end
 
       def supports_where_rules?(obj)
-        WHERE_RULE_TYPES.any? { |t| obj.is_a?(t) }
+        obj.is_a?(Model::HasWhereRules)
       end
 
       # Safe accessor methods that return nil instead of NoMethodError

data/lib/expressir/express/streaming_builder.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "error"
-require_relative "builder"
-
 module Expressir
   module Express
     # Streaming builder that receives parse events from Parsanol native parser.

data/lib/expressir/express/transformer/remark_handling.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "parsanol"
-require "set"
 
 module Expressir
   module Express
@@ -31,8 +30,7 @@ module Expressir
       # @param remarks [Array<Hash>] The extracted remarks
       def attach_remarks(model, remarks)
         # Group remarks by their tag
-        tagged_remarks = remarks.select { |r| r[:tag] }
-        untagged_remarks = remarks.reject { |r| r[:tag] }
+        tagged_remarks, untagged_remarks = remarks.partition { |r| r[:tag] }
 
         # Process tagged remarks
         tagged_remarks.each do |remark|