expressir 2.3.7 → 2.4.1

data/lib/expressir/express/remark_scanner.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Scans EXPRESS source text and extracts all remarks (tail and embedded),
+    # correctly tracking embedded-remark nesting so that inline `--` inside
+    # a `(* ... *)` documentation block is NOT mistaken for a tail remark.
+    #
+    # Single responsibility: source string -> Array<Remark>. The scanner has
+    # no knowledge of the model; attaching remarks to model elements is the
+    # job of {RemarkAttacher}.
+    #
+    # Reference: ISO 10303-11 §7.1.6.
+    #   tail_remark     = '--' [ remark_tag ] { <any char except newline> } '\n'
+    #   embedded_remark = '(*' [ remark_tag ]
+    #                       { <any char> | nested embedded_remark } '*)'
+    #
+    # The scanner is a single-pass state machine with four states:
+    #   :top          — outside any remark/string; look for `--`, `(*`, or `'`.
+    #   :in_tail       — inside a tail remark; look only for the next newline.
+    #   :in_embedded   — inside one or more nested embedded remarks; look
+    #                    for the matching `*)` or a nested `(*`, ignoring `--`.
+    #   :in_string     — inside a simple string literal; look for the closing
+    #                    `'`, treating `''` as an escaped quote (ISO 10303-11
+    #                    §7.1.6.5). `--` and `(*` inside a string are content.
+    class RemarkScanner
+      # Immutable value object describing a single extracted remark.
+      # `position` is the byte offset of the opener (`--` or `(*`).
+      # `end_position` is the byte offset one past the closer (the newline
+      # byte for tail remarks, the byte after `*)` for embedded remarks).
+      # `[position, end_position)` is the full byte range the remark
+      # occupied in the original source.
+      Remark = Struct.new(:position, :end_position, :line, :text, :tag,
+                          :format, keyword_init: true) do
+        def tail?
+          format == Model::RemarkFormat::TAIL
+        end
+
+        def embedded?
+          format == Model::RemarkFormat::EMBEDDED
+        end
+
+        def tagged?
+          !tag.nil? && !tag.empty?
+        end
+
+        # Byte length of the full remark (opener + content + closer).
+        def bytesize
+          end_position - position
+        end
+      end
+
+      # Lexical markers recognised by EXPRESS.
+      TAIL_MARKER = "--"
+      EMBEDDED_OPEN = "(*"
+      EMBEDDED_CLOSE = "*)"
+      STRING_QUOTE = "'"
+
+      # Byte value of `)` — used to detect the `(*)` overlap pattern where
+      # the embedded-remark opener `(*` and closer `*)` share the `*`.
+      RPAREN_BYTE = ")".ord
+
+      # Tag forms (see ISO 10303-11 §7.1.6.3 Remark tag).
+      # IP-style informal-proposition tags are recognised only in tail remarks
+      # (matching historic behaviour); embedded remarks use the quoted form.
+      INFORMAL_PROPOSITION_TAG = /\A(IP\d+):\s*(.*)\z/m
+      QUOTED_TAG = /\A"([^"]*)"\s*(.*)\z/m
+
+      # @param source [String] EXPRESS source text (any encoding).
+      def initialize(source)
+        @source = source
+        @source_bytes = source.b
+        @line_map = LineMap.new(@source_bytes)
+      end
+
+      # Returns an Array of {Remark}, in source order (by byte position).
+      # @return [Array<Remark>]
+      def scan
+        run_state_machine
+      end
+
+      alias call scan
+
+      private
+
+      # Detects the `(*)` overlap pattern: opener `(*` (positions pos, pos+1)
+      # and closer `*)` (positions pos+1, pos+2) share the `*` character.
+      # Per strict ISO 10303-11 §7.1.6 grammar the opener and closer are
+      # distinct tokens that cannot share characters, so `(*)` is technically
+      # invalid; we accept it as a 3-character empty embedded remark for
+      # robustness (issue #126).
+      def empty_overlap_remark?(pos)
+        pos + 2 < @source_bytes.bytesize &&
+          @source_bytes.getbyte(pos + 2) == RPAREN_BYTE
+      end
+
+      def run_state_machine
+        remarks = []
+        src = @source_bytes
+        pos = 0
+
+        state = :top
+        embedded_depth = 0
+        tail_start = nil      # byte position of first content char after "--"
+        tail_line = nil
+        embedded_start = nil  # byte position of first content char after "(*"
+        embedded_line = nil
+
+        while pos < src.bytesize
+          case state
+          when :in_tail
+            nl = src.index("\n", pos)
+            if nl
+              emit_tail(remarks, tail_start, nl, tail_line)
+              pos = nl + 1
+              state = :top
+            else
+              emit_tail(remarks, tail_start, src.bytesize, tail_line)
+              return remarks
+            end
+          when :in_embedded
+            closing = src.index(EMBEDDED_CLOSE, pos)
+            nesting = src.index(EMBEDDED_OPEN, pos)
+            if nesting && (!closing || nesting < closing)
+              if empty_overlap_remark?(nesting)
+                # `(*)` opens and closes immediately; depth unchanged.
+                pos = nesting + 3
+              else
+                embedded_depth += 1
+                pos = nesting + EMBEDDED_OPEN.bytesize
+              end
+            elsif closing
+              embedded_depth -= 1
+              pos = closing + EMBEDDED_CLOSE.bytesize
+              if embedded_depth.zero?
+                emit_embedded(remarks, embedded_start, closing, embedded_line)
+                state = :top
+              end
+            else
+              # Unclosed embedded remark — terminate scan gracefully.
+              return remarks
+            end
+          when :in_string
+            # ISO 10303-11 §7.1.6.5: `''` inside a string is an escaped quote.
+            # Find the next `'`; if followed by another `'`, skip both; else
+            # the string closes.
+            quote_pos = src.index(STRING_QUOTE, pos)
+            if quote_pos.nil?
+              # Unclosed string — malformed EXPRESS; terminate scan gracefully.
+              return remarks
+            end
+
+            if quote_pos + 1 < src.bytesize && src.getbyte(quote_pos + 1) == STRING_QUOTE.ord
+              pos = quote_pos + 2
+            else
+              pos = quote_pos + 1
+              state = :top
+            end
+          else # :top
+            next_embed = src.index(EMBEDDED_OPEN, pos)
+            next_tail = src.index(TAIL_MARKER, pos)
+            next_quote = src.index(STRING_QUOTE, pos)
+            if next_embed.nil? && next_tail.nil? && next_quote.nil?
+              return remarks
+            elsif next_quote && (next_embed.nil? || next_quote < next_embed) &&
+                (next_tail.nil? || next_quote < next_tail)
+              # String opener wins — enter :in_string and skip the quote.
+              pos = next_quote + STRING_QUOTE.bytesize
+              state = :in_string
+            elsif next_embed && (next_tail.nil? || next_embed <= next_tail)
+              if empty_overlap_remark?(next_embed)
+                # Standalone `(*)`: empty embedded remark, opener and closer
+                # share the `*`. Emit with empty content and continue.
+                emit_embedded(remarks, next_embed + EMBEDDED_OPEN.bytesize,
+                              next_embed + EMBEDDED_OPEN.bytesize,
+                              @line_map.line_number(next_embed))
+                pos = next_embed + 3
+                state = :top
+              else
+                embedded_start = next_embed + EMBEDDED_OPEN.bytesize
+                embedded_line = @line_map.line_number(next_embed)
+                embedded_depth = 1
+                state = :in_embedded
+                pos = next_embed + EMBEDDED_OPEN.bytesize
+              end
+            else
+              tail_start = next_tail + TAIL_MARKER.bytesize
+              tail_line = @line_map.line_number(next_tail)
+              state = :in_tail
+              pos = next_tail + TAIL_MARKER.bytesize
+            end
+          end
+        end
+
+        remarks
+      end
+
+      def emit_tail(remarks, content_start, content_end, line)
+        # Slice content from the original (UTF-8) source so the returned text
+        # preserves the source encoding. Byte offsets come from the BINARY copy.
+        raw = @source.byteslice(content_start...content_end)
+        tag, content = parse_tail(raw.strip)
+        remarks << Remark.new(
+          position: content_start - TAIL_MARKER.bytesize,
+          end_position: content_end,
+          line: line,
+          text: content,
+          tag: tag,
+          format: Model::RemarkFormat::TAIL,
+        )
+      end
+
+      def emit_embedded(remarks, content_start, content_end, line)
+        raw = @source.byteslice(content_start...content_end)
+        tag, content = parse_embedded(raw)
+        remarks << Remark.new(
+          position: content_start - EMBEDDED_OPEN.bytesize,
+          end_position: content_end + EMBEDDED_CLOSE.bytesize,
+          line: line,
+          text: content,
+          tag: tag,
+          format: Model::RemarkFormat::EMBEDDED,
+        )
+      end
+
+      # Tail remarks support both quoted tags ("schema.entity" ...) and the
+      # shorthand IP-tag form (IP1: ...). Returns [tag, content]; for untagged
+      # remarks tag is nil and content is the stripped text.
+      def parse_tail(text)
+        match = text.match(INFORMAL_PROPOSITION_TAG) || text.match(QUOTED_TAG)
+        match ? match.captures : [nil, text]
+      end
+
+      # Embedded remarks only use the quoted tag form.
+      def parse_embedded(content)
+        stripped = content.strip
+        if (m = stripped.match(QUOTED_TAG))
+          [m[1], m[2]]
+        else
+          [nil, stripped]
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express/schema_block_scanner.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Splits multi-schema EXPRESS source into individual schema blocks.
+    #
+    # A hand-rolled state machine that skips comments (`(* ... *)`) and
+    # string literals, tracks SCHEMA/END_SCHEMA depth, and returns one
+    # block per schema declaration. Used by `Parser.from_exp_streaming`
+    # to parse each schema independently with a fresh memory arena.
+    #
+    # Extracted from the outer Parser class (TODO.bugs/09) so block-
+    # scanning logic lives in one focused module behind one interface.
+    module SchemaBlockScanner
+      END_SCHEMA_KEYWORD = "END_SCHEMA"
+      SCHEMA_KEYWORD = "SCHEMA"
+      WHITESPACE_CHARS = [" ", "\t", "\n", "\r"].freeze
+
+      module_function
+
+      # Extract individual schema blocks from EXPRESS source.
+      #
+      # Uses a state machine to properly handle nested comments and strings.
+      #
+      # @param source [String] EXPRESS source
+      # @return [Array<Hash>] Array of {name:, source:, start_pos:, end_pos:}
+      #   for each schema block found.
+      def extract_schema_blocks(source)
+        blocks = []
+        pos = 0
+        len = source.length
+
+        while pos < len
+          pos = skip_ws_and_comments(source, pos)
+          break if pos >= len
+
+          if source[pos..].start_with?(SCHEMA_KEYWORD)
+            result = parse_schema_block(source, pos)
+            if result
+              blocks << result
+              pos = result[:end_pos]
+              next
+            end
+          end
+
+          pos += 1
+        end
+
+        blocks
+      end
+
+      def parse_schema_block(source, start_pos)
+        return nil unless source[start_pos..].start_with?(SCHEMA_KEYWORD)
+
+        pos = start_pos + SCHEMA_KEYWORD.length
+        pos = skip_spaces(source, pos)
+
+        name_start = pos
+        while pos < source.length && (source[pos] =~ /[a-zA-Z0-9_]/ || source[pos] == "_")
+          pos += 1
+        end
+        schema_name = source[name_start...pos]
+        return nil if schema_name.empty?
+
+        depth = 1
+        search_pos = pos
+        end_pos = nil
+
+        while search_pos < source.length
+          if source[search_pos] == '"'
+            search_pos += 1
+            while search_pos < source.length && source[search_pos] != '"'
+              search_pos += 1
+            end
+            search_pos += 1
+          elsif source[search_pos] == "(" && source[search_pos + 1] == "*"
+            search_pos += 2
+            while search_pos < source.length && !(source[search_pos] == "*" && source[search_pos + 1] == ")")
+              search_pos += 1
+            end
+            search_pos += 2
+          elsif source[search_pos..].start_with?(END_SCHEMA_KEYWORD)
+            depth -= 1
+            if depth.zero?
+              end_pos = search_pos + END_SCHEMA_KEYWORD.length
+              pos = end_pos
+              pos = skip_spaces(source, pos)
+              pos += 1 if source[pos] == ";"
+              break
+            end
+            search_pos += END_SCHEMA_KEYWORD.length
+          else
+            search_pos += 1
+          end
+        end
+
+        return nil unless end_pos
+
+        {
+          name: schema_name,
+          source: source[start_pos...end_pos],
+          start_pos: start_pos,
+          end_pos: pos,
+        }
+      end
+
+      def skip_spaces(source, pos)
+        while pos < source.length && WHITESPACE_CHARS.include?(source[pos])
+          pos += 1
+        end
+        pos
+      end
+
+      def skip_ws_and_comments(source, pos)
+        len = source.length
+        while pos < len
+          c = source[pos]
+          if WHITESPACE_CHARS.include?(c)
+            pos += 1
+          elsif c == "(" && source[pos + 1] == "*"
+            pos += 2
+            while pos < len - 1 && !(source[pos] == "*" && source[pos + 1] == ")")
+              pos += 1
+            end
+            pos += 2
+          else
+            break
+          end
+        end
+        pos
+      end
+
+      private_class_method :parse_schema_block, :skip_spaces, :skip_ws_and_comments
+    end
+  end
+end

data/lib/expressir/express/schema_source_formatter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Pre-built Formatter subclass for Schema#source.
+    #
+    # Combines SchemaHeadFormatter (renders the SCHEMA header block) with
+    # HyperlinkFormatter (renders cross-references as hyperlinks). Allocated
+    # once at load time rather than per #source call.
+    class SchemaSourceFormatter < Formatter
+      include SchemaHeadFormatter
+      include HyperlinkFormatter
+    end
+  end
+end

data/lib/expressir/express/scope_resolver.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Resolves the containing scope for a remark line.
+    #
+    # One interface (`containing_scope_for`), four strategies behind it:
+    #   1. O(1) lookup via a precomputed line→scope-name map
+    #   2. O(n) position-based fallback against the node index
+    #   3. Source-text scan for entity/type/rule boundaries
+    #   4. Schema-wide search by name
+    #
+    # Extracted from RemarkAttacher to deepen the seam: scope lookup is the
+    # only concern here, so changes to scope-detection heuristics land in
+    # one module and the attacher stops carrying the how.
+    class ScopeResolver
+      # Match declarations of the scopes we track.
+      SCOPE_OPEN_PATTERNS = {
+        Model::Declarations::Schema => /^\s*SCHEMA\s+(\w+)/i,
+        Model::Declarations::Function => /^\s*FUNCTION\s+(\w+)/i,
+        Model::Declarations::Procedure => /^\s*PROCEDURE\s+(\w+)/i,
+        Model::Declarations::Rule => /^\s*RULE\s+(\w+)/i,
+        Model::Declarations::Entity => /^\s*ENTITY\s+(\w+)/i,
+        Model::Declarations::Type => /^\s*TYPE\s+(\w+)/i,
+      }.freeze
+
+      SCOPE_CLOSE_PATTERNS = {
+        Model::Declarations::Schema => /END_SCHEMA/i,
+        Model::Declarations::Function => /END_FUNCTION/i,
+        Model::Declarations::Procedure => /END_PROCEDURE/i,
+        Model::Declarations::Rule => /END_RULE/i,
+        Model::Declarations::Entity => /END_ENTITY/i,
+        Model::Declarations::Type => /END_TYPE/i,
+      }.freeze
+
+      # Map scope node class → schema-level collection accessor on Schema.
+      SCHEMA_COLLECTION_ACCESSOR = {
+        Model::Declarations::Entity => lambda(&:entities),
+        Model::Declarations::Type => lambda(&:types),
+        Model::Declarations::Rule => lambda(&:rules),
+      }.freeze
+
+      # Schema-level collections searched when resolving a scope name to a
+      # model node. Sourced from Model::Declarations::Schema::SCOPE_DECL_COLLECTIONS
+      # so the "which Schema collections hold named scopes" fact lives on
+      # the model, not duplicated here.
+
+      def initialize(source:, model:, nodes_with_positions:)
+        @source = source
+        @model = model
+        @nodes_with_positions = nodes_with_positions
+        @scope_map = nil
+      end
+
+      # Returns the innermost Model::ScopeContainer whose source span contains
+      # the given 1-based remark line, or nil if none is found.
+      def containing_scope_for(remark_line)
+        scope = find_by_name(remark_line)
+        scope ||= find_by_position(remark_line)
+        scope
+      end
+
+      # Source-text fallback: scan backwards from remark_line for the
+      # entity/type/rule declaration that contains it. Used by the attacher
+      # when both name- and position-based lookup miss (e.g. for IP tags
+      # whose target is a Type/Entity/Rule declared on a previous line).
+      def find_by_source_text(remark_line)
+        entity_state = { line: nil, name: nil }
+        type_state = { line: nil, name: nil }
+        rule_state = { line: nil, name: nil }
+
+        @source.lines.each_with_index do |line, idx|
+          line_num = idx + 1
+
+          case line
+          when /^\s*ENTITY\s+(\w+)/i
+            entity_state = { line: line_num, name: $1 }
+          when /^\s*END_ENTITY/i
+            return find_in_schema(Model::Declarations::Entity, entity_state[:name]) if span_contains?(entity_state, remark_line, line_num)
+
+            entity_state = { line: nil, name: nil }
+          when /^\s*TYPE\s+(\w+)/i
+            type_state = { line: line_num, name: $1 }
+          when /^\s*END_TYPE/i
+            return find_in_schema(Model::Declarations::Type, type_state[:name]) if span_contains?(type_state, remark_line, line_num)
+
+            type_state = { line: nil, name: nil }
+          when /^\s*RULE\s+(\w+)/i
+            rule_state = { line: line_num, name: $1 }
+          when /^\s*END_RULE/i
+            return find_in_schema(Model::Declarations::Rule, rule_state[:name]) if span_contains?(rule_state, remark_line, line_num)
+
+            rule_state = { line: nil, name: nil }
+          end
+        end
+
+        nil
+      end
+
+      private
+
+      def span_contains?(state, remark_line, end_line)
+        state[:line] && remark_line >= state[:line] && remark_line <= end_line
+      end
+
+      # --- Strategy 1: precomputed line → scope-name map ---
+
+      def find_by_name(remark_line)
+        scope_name = scope_map[remark_line]
+        return nil unless scope_name
+        return nil unless @model
+
+        @model.schemas.each do |schema|
+          return schema if schema.id == scope_name
+
+          Model::Declarations::Schema::SCOPE_DECL_COLLECTIONS.each do |decl_type|
+            collection = schema.public_send(decl_type)
+            next unless collection.is_a?(Array)
+
+            found = collection.find { |n| n.id == scope_name }
+            return found if found
+          end
+        end
+
+        nil
+      end
+
+      # Lazily build, on first lookup, the line → scope-name table.
+      # O(file_lines) scan; subsequent lookups are O(1).
+      def scope_map
+        @scope_map ||= build_scope_map
+      end
+
+      def build_scope_map
+        lines = @source.lines
+        map = {}
+        return map if lines.empty?
+
+        stack = [] # array of { type: Class, name: String }
+
+        lines.each_with_index do |line, idx|
+          line_num = idx + 1
+
+          SCOPE_OPEN_PATTERNS.each do |klass, pattern|
+            stack << { type: klass, name: $1 } if line =~ pattern
+          end
+
+          SCOPE_CLOSE_PATTERNS.each do |klass, pattern|
+            next unless line&.match?(pattern)
+            next unless stack.last && stack.last[:type] == klass
+
+            stack.pop
+          end
+
+          map[line_num] = stack.last&.dig(:name)
+        end
+
+        map
+      end
+
+      # --- Strategy 2: position-based fallback against the node index ---
+
+      def find_by_position(remark_line)
+        containing = @nodes_with_positions.select do |n|
+          n[:line] && n[:end_line] &&
+            remark_line >= n[:line] && remark_line <= n[:end_line] &&
+            !n[:node].is_a?(Model::Repository) && !n[:node].is_a?(Model::Cache)
+        end
+
+        containing.reverse_each do |n|
+          return n[:node] if n[:node].is_a?(Model::ScopeContainer)
+        end
+
+        nil
+      end
+
+      # --- Schema-wide name search (used by source-text fallback) ---
+
+      def find_in_schema(node_class, name)
+        return nil unless @model && name
+
+        accessor = SCHEMA_COLLECTION_ACCESSOR[node_class]
+        return nil unless accessor
+
+        @model.schemas.each do |schema|
+          found = accessor.call(schema)&.find { |n| n.id == name }
+          return found if found
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/expressir/express/source_formatter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Pre-built Formatter subclass that mixes in HyperlinkFormatter.
+    #
+    # Used by ModelElement#source to format an element back into EXPRESS
+    # source text with hyperlinks. Allocated once at load time rather than
+    # per #source call (which was the previous behaviour — wasteful and
+    # rebuilt the format_registry on every call).
+    class SourceFormatter < Formatter
+      include HyperlinkFormatter
+    end
+  end
+end