expressir 2.4.0 → 2.4.2

data/lib/expressir/express/remark_scanner.rb

@@ -15,35 +15,50 @@ module Expressir
     #   embedded_remark = '(*' [ remark_tag ]
     #                       { <any char> | nested embedded_remark } '*)'
     #
-    # The scanner is a single-pass state machine with three states:
-    #   :top          — outside any remark; look for `--` or `(*`.
+    # The scanner is a single-pass state machine with four states:
+    #   :top          — outside any remark/string; look for `--`, `(*`, or `'`.
     #   :in_tail       — inside a tail remark; look only for the next newline.
     #   :in_embedded   — inside one or more nested embedded remarks; look
     #                    for the matching `*)` or a nested `(*`, ignoring `--`.
+    #   :in_string     — inside a simple string literal; look for the closing
+    #                    `'`, treating `''` as an escaped quote (ISO 10303-11
+    #                    §7.1.6.5). `--` and `(*` inside a string are content.
     class RemarkScanner
       # Immutable value object describing a single extracted remark.
-      Remark = Struct.new(:position, :line, :text, :tag, :format, keyword_init: true) do
+      # `position` is the byte offset of the opener (`--` or `(*`).
+      # `end_position` is the byte offset one past the closer (the newline
+      # byte for tail remarks, the byte after `*)` for embedded remarks).
+      # `[position, end_position)` is the full byte range the remark
+      # occupied in the original source.
+      Remark = Struct.new(:position, :end_position, :line, :text, :tag,
+                          :format, keyword_init: true) do
         def tail?
-          format == TAIL_FORMAT
+          format == Model::RemarkFormat::TAIL
         end
 
         def embedded?
-          format == EMBEDDED_FORMAT
+          format == Model::RemarkFormat::EMBEDDED
         end
 
         def tagged?
           !tag.nil? && !tag.empty?
         end
-      end
 
-      # Format discriminator values.
-      TAIL_FORMAT = "tail"
-      EMBEDDED_FORMAT = "embedded"
+        # Byte length of the full remark (opener + content + closer).
+        def bytesize
+          end_position - position
+        end
+      end
 
       # Lexical markers recognised by EXPRESS.
       TAIL_MARKER = "--"
       EMBEDDED_OPEN = "(*"
       EMBEDDED_CLOSE = "*)"
+      STRING_QUOTE = "'"
+
+      # Byte value of `)` — used to detect the `(*)` overlap pattern where
+      # the embedded-remark opener `(*` and closer `*)` share the `*`.
+      RPAREN_BYTE = ")".ord
 
       # Tag forms (see ISO 10303-11 §7.1.6.3 Remark tag).
       # IP-style informal-proposition tags are recognised only in tail remarks
@@ -68,6 +83,17 @@ module Expressir
 
       private
 
+      # Detects the `(*)` overlap pattern: opener `(*` (positions pos, pos+1)
+      # and closer `*)` (positions pos+1, pos+2) share the `*` character.
+      # Per strict ISO 10303-11 §7.1.6 grammar the opener and closer are
+      # distinct tokens that cannot share characters, so `(*)` is technically
+      # invalid; we accept it as a 3-character empty embedded remark for
+      # robustness (issue #126).
+      def empty_overlap_remark?(pos)
+        pos + 2 < @source_bytes.bytesize &&
+          @source_bytes.getbyte(pos + 2) == RPAREN_BYTE
+      end
+
       def run_state_machine
         remarks = []
         src = @source_bytes
@@ -96,8 +122,13 @@ module Expressir
             closing = src.index(EMBEDDED_CLOSE, pos)
             nesting = src.index(EMBEDDED_OPEN, pos)
             if nesting && (!closing || nesting < closing)
-              embedded_depth += 1
-              pos = nesting + EMBEDDED_OPEN.bytesize
+              if empty_overlap_remark?(nesting)
+                # `(*)` opens and closes immediately; depth unchanged.
+                pos = nesting + 3
+              else
+                embedded_depth += 1
+                pos = nesting + EMBEDDED_OPEN.bytesize
+              end
             elsif closing
               embedded_depth -= 1
               pos = closing + EMBEDDED_CLOSE.bytesize
@@ -109,17 +140,49 @@ module Expressir
               # Unclosed embedded remark — terminate scan gracefully.
               return remarks
             end
+          when :in_string
+            # ISO 10303-11 §7.1.6.5: `''` inside a string is an escaped quote.
+            # Find the next `'`; if followed by another `'`, skip both; else
+            # the string closes.
+            quote_pos = src.index(STRING_QUOTE, pos)
+            if quote_pos.nil?
+              # Unclosed string — malformed EXPRESS; terminate scan gracefully.
+              return remarks
+            end
+
+            if quote_pos + 1 < src.bytesize && src.getbyte(quote_pos + 1) == STRING_QUOTE.ord
+              pos = quote_pos + 2
+            else
+              pos = quote_pos + 1
+              state = :top
+            end
           else # :top
             next_embed = src.index(EMBEDDED_OPEN, pos)
             next_tail = src.index(TAIL_MARKER, pos)
-            if next_embed.nil? && next_tail.nil?
+            next_quote = src.index(STRING_QUOTE, pos)
+            if next_embed.nil? && next_tail.nil? && next_quote.nil?
               return remarks
+            elsif next_quote && (next_embed.nil? || next_quote < next_embed) &&
+                (next_tail.nil? || next_quote < next_tail)
+              # String opener wins — enter :in_string and skip the quote.
+              pos = next_quote + STRING_QUOTE.bytesize
+              state = :in_string
             elsif next_embed && (next_tail.nil? || next_embed <= next_tail)
-              embedded_start = next_embed + EMBEDDED_OPEN.bytesize
-              embedded_line = @line_map.line_number(next_embed)
-              embedded_depth = 1
-              state = :in_embedded
-              pos = next_embed + EMBEDDED_OPEN.bytesize
+              if empty_overlap_remark?(next_embed)
+                # Standalone `(*)`: empty embedded remark, opener and closer
+                # share the `*`. Emit with empty content and continue.
+                emit_embedded(remarks, next_embed + EMBEDDED_OPEN.bytesize,
+                              next_embed + EMBEDDED_OPEN.bytesize,
+                              @line_map.line_number(next_embed))
+                pos = next_embed + 3
+                state = :top
+              else
+                embedded_start = next_embed + EMBEDDED_OPEN.bytesize
+                embedded_line = @line_map.line_number(next_embed)
+                embedded_depth = 1
+                state = :in_embedded
+                pos = next_embed + EMBEDDED_OPEN.bytesize
+              end
             else
               tail_start = next_tail + TAIL_MARKER.bytesize
               tail_line = @line_map.line_number(next_tail)
@@ -139,10 +202,11 @@ module Expressir
         tag, content = parse_tail(raw.strip)
         remarks << Remark.new(
           position: content_start - TAIL_MARKER.bytesize,
+          end_position: content_end,
           line: line,
           text: content,
           tag: tag,
-          format: TAIL_FORMAT,
+          format: Model::RemarkFormat::TAIL,
         )
       end
 
@@ -151,10 +215,11 @@ module Expressir
         tag, content = parse_embedded(raw)
         remarks << Remark.new(
           position: content_start - EMBEDDED_OPEN.bytesize,
+          end_position: content_end + EMBEDDED_CLOSE.bytesize,
           line: line,
           text: content,
           tag: tag,
-          format: EMBEDDED_FORMAT,
+          format: Model::RemarkFormat::EMBEDDED,
         )
       end
 

data/lib/expressir/express/schema_block_scanner.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Splits multi-schema EXPRESS source into individual schema blocks.
+    #
+    # A hand-rolled state machine that skips comments (`(* ... *)`) and
+    # string literals, tracks SCHEMA/END_SCHEMA depth, and returns one
+    # block per schema declaration. Retained for the future streaming
+    # parse path (see TODO.max-perf/02) that parses each schema
+    # independently with a fresh memory arena.
+    #
+    # Extracted from the outer Parser class (TODO.bugs/09) so block-
+    # scanning logic lives in one focused module behind one interface.
+    module SchemaBlockScanner
+      END_SCHEMA_KEYWORD = "END_SCHEMA"
+      SCHEMA_KEYWORD = "SCHEMA"
+      WHITESPACE_CHARS = [" ", "\t", "\n", "\r"].freeze
+
+      module_function
+
+      # Extract individual schema blocks from EXPRESS source.
+      #
+      # Uses a state machine to properly handle nested comments and strings.
+      #
+      # @param source [String] EXPRESS source
+      # @return [Array<Hash>] Array of {name:, source:, start_pos:, end_pos:}
+      #   for each schema block found.
+      def extract_schema_blocks(source)
+        blocks = []
+        pos = 0
+        len = source.length
+
+        while pos < len
+          pos = skip_ws_and_comments(source, pos)
+          break if pos >= len
+
+          if source[pos..].start_with?(SCHEMA_KEYWORD)
+            result = parse_schema_block(source, pos)
+            if result
+              blocks << result
+              pos = result[:end_pos]
+              next
+            end
+          end
+
+          pos += 1
+        end
+
+        blocks
+      end
+
+      def parse_schema_block(source, start_pos)
+        return nil unless source[start_pos..].start_with?(SCHEMA_KEYWORD)
+
+        pos = start_pos + SCHEMA_KEYWORD.length
+        pos = skip_spaces(source, pos)
+
+        name_start = pos
+        while pos < source.length && (source[pos] =~ /[a-zA-Z0-9_]/ || source[pos] == "_")
+          pos += 1
+        end
+        schema_name = source[name_start...pos]
+        return nil if schema_name.empty?
+
+        depth = 1
+        search_pos = pos
+        end_pos = nil
+
+        while search_pos < source.length
+          if source[search_pos] == '"'
+            search_pos += 1
+            while search_pos < source.length && source[search_pos] != '"'
+              search_pos += 1
+            end
+            search_pos += 1
+          elsif source[search_pos] == "(" && source[search_pos + 1] == "*"
+            search_pos += 2
+            while search_pos < source.length && !(source[search_pos] == "*" && source[search_pos + 1] == ")")
+              search_pos += 1
+            end
+            search_pos += 2
+          elsif source[search_pos..].start_with?(END_SCHEMA_KEYWORD)
+            depth -= 1
+            if depth.zero?
+              end_pos = search_pos + END_SCHEMA_KEYWORD.length
+              pos = end_pos
+              pos = skip_spaces(source, pos)
+              pos += 1 if source[pos] == ";"
+              break
+            end
+            search_pos += END_SCHEMA_KEYWORD.length
+          else
+            search_pos += 1
+          end
+        end
+
+        return nil unless end_pos
+
+        {
+          name: schema_name,
+          source: source[start_pos...end_pos],
+          start_pos: start_pos,
+          end_pos: pos,
+        }
+      end
+
+      def skip_spaces(source, pos)
+        while pos < source.length && WHITESPACE_CHARS.include?(source[pos])
+          pos += 1
+        end
+        pos
+      end
+
+      def skip_ws_and_comments(source, pos)
+        len = source.length
+        while pos < len
+          c = source[pos]
+          if WHITESPACE_CHARS.include?(c)
+            pos += 1
+          elsif c == "(" && source[pos + 1] == "*"
+            pos += 2
+            while pos < len - 1 && !(source[pos] == "*" && source[pos + 1] == ")")
+              pos += 1
+            end
+            pos += 2
+          else
+            break
+          end
+        end
+        pos
+      end
+
+      private_class_method :parse_schema_block, :skip_spaces, :skip_ws_and_comments
+    end
+  end
+end

data/lib/expressir/express/schema_source_formatter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Pre-built Formatter subclass for Schema#source.
+    #
+    # Combines SchemaHeadFormatter (renders the SCHEMA header block) with
+    # HyperlinkFormatter (renders cross-references as hyperlinks). Allocated
+    # once at load time rather than per #source call.
+    class SchemaSourceFormatter < Formatter
+      include SchemaHeadFormatter
+      include HyperlinkFormatter
+    end
+  end
+end

data/lib/expressir/express/scope_resolver.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Resolves the containing scope for a remark line.
+    #
+    # One interface (`containing_scope_for`), four strategies behind it:
+    #   1. O(1) lookup via a precomputed line→scope-name map
+    #   2. O(n) position-based fallback against the node index
+    #   3. Source-text scan for entity/type/rule boundaries
+    #   4. Schema-wide search by name
+    #
+    # Extracted from RemarkAttacher to deepen the seam: scope lookup is the
+    # only concern here, so changes to scope-detection heuristics land in
+    # one module and the attacher stops carrying the how.
+    class ScopeResolver
+      # Match declarations of the scopes we track.
+      SCOPE_OPEN_PATTERNS = {
+        Model::Declarations::Schema => /^\s*SCHEMA\s+(\w+)/i,
+        Model::Declarations::Function => /^\s*FUNCTION\s+(\w+)/i,
+        Model::Declarations::Procedure => /^\s*PROCEDURE\s+(\w+)/i,
+        Model::Declarations::Rule => /^\s*RULE\s+(\w+)/i,
+        Model::Declarations::Entity => /^\s*ENTITY\s+(\w+)/i,
+        Model::Declarations::Type => /^\s*TYPE\s+(\w+)/i,
+      }.freeze
+
+      SCOPE_CLOSE_PATTERNS = {
+        Model::Declarations::Schema => /END_SCHEMA/i,
+        Model::Declarations::Function => /END_FUNCTION/i,
+        Model::Declarations::Procedure => /END_PROCEDURE/i,
+        Model::Declarations::Rule => /END_RULE/i,
+        Model::Declarations::Entity => /END_ENTITY/i,
+        Model::Declarations::Type => /END_TYPE/i,
+      }.freeze
+
+      # Map scope node class → schema-level collection accessor on Schema.
+      SCHEMA_COLLECTION_ACCESSOR = {
+        Model::Declarations::Entity => lambda(&:entities),
+        Model::Declarations::Type => lambda(&:types),
+        Model::Declarations::Rule => lambda(&:rules),
+      }.freeze
+
+      # Schema-level collections searched when resolving a scope name to a
+      # model node. Sourced from Model::Declarations::Schema::SCOPE_DECL_COLLECTIONS
+      # so the "which Schema collections hold named scopes" fact lives on
+      # the model, not duplicated here.
+
+      def initialize(source:, model:, nodes_with_positions:)
+        @source = source
+        @model = model
+        @nodes_with_positions = nodes_with_positions
+        @scope_map = nil
+        @source_lines = nil
+        @position_buckets = nil
+      end
+
+      # Line-band width for the position fallback index.
+      BUCKET_LINES = 1024
+
+      # Returns the innermost Model::ScopeContainer whose source span contains
+      # the given 1-based remark line, or nil if none is found.
+      def containing_scope_for(remark_line)
+        scope = find_by_name(remark_line)
+        scope ||= find_by_position(remark_line)
+        scope
+      end
+
+      # Source-text fallback: scan backwards from remark_line for the
+      # entity/type/rule declaration that contains it. Used by the attacher
+      # when both name- and position-based lookup miss (e.g. for IP tags
+      # whose target is a Type/Entity/Rule declared on a previous line).
+      def find_by_source_text(remark_line)
+        entity_state = { line: nil, name: nil }
+        type_state = { line: nil, name: nil }
+        rule_state = { line: nil, name: nil }
+
+        source_lines.each_with_index do |line, idx|
+          line_num = idx + 1
+
+          case line
+          when /^\s*ENTITY\s+(\w+)/i
+            entity_state = { line: line_num, name: $1 }
+          when /^\s*END_ENTITY/i
+            return find_in_schema(Model::Declarations::Entity, entity_state[:name]) if span_contains?(entity_state, remark_line, line_num)
+
+            entity_state = { line: nil, name: nil }
+          when /^\s*TYPE\s+(\w+)/i
+            type_state = { line: line_num, name: $1 }
+          when /^\s*END_TYPE/i
+            return find_in_schema(Model::Declarations::Type, type_state[:name]) if span_contains?(type_state, remark_line, line_num)
+
+            type_state = { line: nil, name: nil }
+          when /^\s*RULE\s+(\w+)/i
+            rule_state = { line: line_num, name: $1 }
+          when /^\s*END_RULE/i
+            return find_in_schema(Model::Declarations::Rule, rule_state[:name]) if span_contains?(rule_state, remark_line, line_num)
+
+            rule_state = { line: nil, name: nil }
+          end
+        end
+
+        nil
+      end
+
+      private
+
+      def span_contains?(state, remark_line, end_line)
+        state[:line] && remark_line >= state[:line] && remark_line <= end_line
+      end
+
+      # --- Strategy 1: precomputed line → scope-name map ---
+
+      def find_by_name(remark_line)
+        scope_name = scope_map[remark_line]
+        return nil unless scope_name
+        return nil unless @model
+
+        @model.schemas.each do |schema|
+          return schema if schema.id == scope_name
+
+          Model::Declarations::Schema::SCOPE_DECL_COLLECTIONS.each do |decl_type|
+            collection = schema.public_send(decl_type)
+            next unless collection.is_a?(Array)
+
+            found = collection.find { |n| n.id == scope_name }
+            return found if found
+          end
+        end
+
+        nil
+      end
+
+      # Lazily build, on first lookup, the line → scope-name table.
+      # O(file_lines) scan; subsequent lookups are O(1).
+      def scope_map
+        @scope_map ||= build_scope_map
+      end
+
+      # Source split into lines, computed once per resolver: both the
+      # scope-map build and find_by_source_text need it, and re-splitting the
+      # whole source per lookup dominates remark attachment on large files.
+      def source_lines
+        @source_lines ||= @source.lines
+      end
+
+      def build_scope_map
+        lines = source_lines
+        map = {}
+        return map if lines.empty?
+
+        stack = [] # array of { type: Class, name: String }
+
+        lines.each_with_index do |line, idx|
+          line_num = idx + 1
+
+          SCOPE_OPEN_PATTERNS.each do |klass, pattern|
+            stack << { type: klass, name: $1 } if line =~ pattern
+          end
+
+          SCOPE_CLOSE_PATTERNS.each do |klass, pattern|
+            next unless line&.match?(pattern)
+            next unless stack.last && stack.last[:type] == klass
+
+            stack.pop
+          end
+
+          map[line_num] = stack.last&.dig(:name)
+        end
+
+        map
+      end
+
+      # --- Strategy 2: position-based fallback against the node index ---
+
+      # Nodes bucketed by 1024-line bands: a node spanning [line, end_line]
+      # is registered in every band it overlaps, so a remark-line lookup only
+      # scans nodes that can possibly contain it. Bucket order preserves the
+      # original index order, so select+reverse_each semantics are unchanged.
+      def position_buckets
+        @position_buckets ||= begin
+          buckets = Hash.new { |h, k| h[k] = [] }
+          @nodes_with_positions.each do |n|
+            next unless n[:line] && n[:end_line]
+
+            ((n[:line] / BUCKET_LINES)..(n[:end_line] / BUCKET_LINES)).each do |b|
+              buckets[b] << n
+            end
+          end
+          buckets
+        end
+      end
+
+      def find_by_position(remark_line)
+        containing = position_buckets[remark_line / BUCKET_LINES].select do |n|
+          remark_line.between?(n[:line], n[:end_line]) &&
+            !n[:node].is_a?(Model::Repository) && !n[:node].is_a?(Model::Cache)
+        end
+
+        containing.reverse_each do |n|
+          return n[:node] if n[:node].is_a?(Model::ScopeContainer)
+        end
+
+        nil
+      end
+
+      # --- Schema-wide name search (used by source-text fallback) ---
+
+      def find_in_schema(node_class, name)
+        return nil unless @model && name
+
+        accessor = SCHEMA_COLLECTION_ACCESSOR[node_class]
+        return nil unless accessor
+
+        @model.schemas.each do |schema|
+          found = accessor.call(schema)&.find { |n| n.id == name }
+          return found if found
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/expressir/express/source_formatter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Pre-built Formatter subclass that mixes in HyperlinkFormatter.
+    #
+    # Used by ModelElement#source to format an element back into EXPRESS
+    # source text with hyperlinks. Allocated once at load time rather than
+    # per #source call (which was the previous behaviour — wasteful and
+    # rebuilt the format_registry on every call).
+    class SourceFormatter < Formatter
+      include HyperlinkFormatter
+    end
+  end
+end