expressir 2.3.7 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecbd13317e56cc0139eb317331134c80ba9f5ae95eaf221f2f5e2046ebe95387
-  data.tar.gz: 86dbdfe6818f5c751d11c6fc4cd81e53a60b38145d3f7d80f5bd1b5ff6869815
+  metadata.gz: 738451ad45c2f5e9268ef7562cbb2c667b90648f0e2df9ac409c79ceeeb6b879
+  data.tar.gz: e4e04ff5ede476581d65882a5e8d6fefba98ced3345337673bf61366e5b96392
 SHA512:
-  metadata.gz: b3b26b8456c55b2a5b342d508a2af8f1e3c46ed23f78b96b5900a036292533c33b68f140248f6c8b552f5ed571adc245fe71acf1159b21feca3aa6c0b7bb354f
-  data.tar.gz: 3d33e2d5c850898a8b451a91cd76a9a85736b7e4c6d846c29e1652dc5e8fbb080bec35b2c55a908c7da68bdb32c5b7940162ea88cd811a519643a5f97db65fcf
+  metadata.gz: 6e9c0b2be28d8fa52f4c64f151e73c345701b2e6a118ee771f1442459c9c97bd5539bf1d799692ad065317849f9e794fb4cdc03a7d55efec1ddf627408f710de
+  data.tar.gz: 44100a53632c094d8cd7d5da781a8f2ab63274f4fcdcad44a0aafaf2e6c94821a5eefa6263c9f4c982876f55aeb6e23f5858cb7ca78decce3489a4c00b50abea

data/lib/expressir/express/line_map.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Maps byte positions in a source string to 1-based line numbers.
+    #
+    # Single source of truth for byte→line lookup across the remark pipeline
+    # (RemarkScanner and RemarkAttacher both rely on it). Builds an offset
+    # table once and answers queries in O(log n) via binary search.
+    class LineMap
+      # Newline byte (ASCII 0x0A).
+      NEWLINE = "\n"
+
+      # @param source_bytes [String] BINARY / ASCII-8BIT encoded source.
+      def initialize(source_bytes)
+        @offsets = build_offsets(source_bytes)
+      end
+
+      # Returns the 1-based line number that contains the given byte position.
+      # Returns 1 for position 0 (or any position before the first newline).
+      # Returns the last line number for positions past the final newline.
+      #
+      # @param byte_pos [Integer, nil]
+      # @return [Integer] 1-based line number; nil input yields 1.
+      def line_number(byte_pos)
+        return 1 if byte_pos.nil? || byte_pos <= 0
+
+        idx = @offsets.bsearch_index { |offset| offset > byte_pos }
+        idx || @offsets.size
+      end
+
+      private
+
+      # Builds the sorted array of byte positions where each line begins.
+      # offsets[0] = 0 (line 1 begins at byte 0).
+      # offsets[i] = (byte position of the i-th newline) + 1 (line i+1 begins).
+      def build_offsets(source_bytes)
+        offsets = [0]
+        i = 0
+        while (i = source_bytes.index(NEWLINE, i))
+          offsets << (i + 1)
+          i += 1
+        end
+        offsets
+      end
+    end
+  end
+end

data/lib/expressir/express/remark_attacher.rb

@@ -48,7 +48,7 @@ module Expressir
       def initialize(source)
         @source = source
         @attached_spans = Set.new
-        @line_cache = {}
+        @line_map = LineMap.new(source.b)
         @model = nil
         @source_lines = nil # cached @source.lines
         @scope_map = nil # cached scope at each line number
@@ -56,7 +56,7 @@ module Expressir
 
       def attach(model)
         @model = model
-        remarks = extract_all_remarks
+        remarks = RemarkScanner.new(@source).scan
 
         # Build nodes_with_positions ONCE for both tagged and untagged remark passes.
         # This avoids double tree walk (381K nodes × 2 = 762K visits) which was
@@ -71,103 +71,22 @@ module Expressir
         @source = nil
         @source_lines = nil
         @scope_map = nil
-        @line_cache = nil
+        @line_map = nil
 
         model
       end
 
       private
 
-      def extract_all_remarks
-        remarks = []
-        byte_position = 0
-
-        @source.each_line.with_index do |line, line_idx|
-          line_bytesize = line.bytesize
-          line_bytes = line.b # Get byte string for indexing
-          if (dash_byte_idx = line_bytes.index("--"))
-            remark_text = line.byteslice((dash_byte_idx + 2)..).strip
-
-            # Check for special patterns like --IP1: content (informal proposition)
-            if remark_text.match?(/^IP\d+:\s*(.*)$/)
-              tag = remark_text[/^(IP\d+):/, 1]
-              content = remark_text[/^IP\d+:\s*(.*)$/, 1]
-              remarks << {
-                position: byte_position + dash_byte_idx,
-                line: line_idx + 1,
-                text: content,
-                tag: tag,
-                format: "tail",
-              }
-            else
-              tag, content = parse_tagged_remark(remark_text)
-              remarks << {
-                position: byte_position + dash_byte_idx,
-                line: line_idx + 1,
-                text: content || remark_text,
-                tag: tag,
-                format: "tail",
-              }
-            end
-          end
-          byte_position += line_bytesize
-        end
-
-        extract_embedded_remarks(remarks)
-        # Sort by position to ensure remarks are processed in source order
-        remarks.sort_by! { |r| r[:position] }
-        remarks
-      end
-
-      def extract_embedded_remarks(remarks)
-        source_bytes = @source.b
-        start_pos = 0
-        while (start_idx = source_bytes.index("(*", start_pos))
-          end_idx = source_bytes.index("*)", start_idx + 2)
-          break unless end_idx
-
-          content = @source.byteslice((start_idx + 2)...end_idx)
-          line_num = get_line_number(start_idx)
-
-          tag, text = parse_tagged_embedded_remark(content)
-
-          remarks << {
-            position: start_idx,
-            line: line_num,
-            text: text,
-            tag: tag,
-            format: "embedded",
-          }
-
-          start_pos = end_idx + 2
-        end
-      end
-
-      def parse_tagged_remark(text)
-        if text.start_with?('"') && (end_quote = text.index('"', 1))
-          [text[1...end_quote], text[(end_quote + 1)..].strip]
-        else
-          [nil, text]
-        end
-      end
-
-      def parse_tagged_embedded_remark(content)
-        stripped = content.strip
-        if stripped.start_with?('"') && (end_quote = stripped.index('"', 1))
-          [stripped[1...end_quote], stripped[(end_quote + 1)..].strip]
-        else
-          [nil, stripped]
-        end
-      end
+      # Remark extraction lives in {RemarkScanner}; this class only attaches.
+      # Line-number lookup is delegated to {LineMap} for O(log n) queries.
 
       def source_lines
         @source_lines ||= @source.lines
       end
 
       def get_line_number(position)
-        return 1 if position.nil? || position.zero?
-
-        @line_cache[position] ||= @source.byteslice(0...position).count("\n") + 1
+        @line_map.line_number(position)
       end
 
       def attach_tagged_remarks(model, remarks, nodes_with_positions)
@@ -180,23 +99,23 @@ module Expressir
         # This is the key optimization that makes scope lookup O(1) per remark
         @scope_map ||= build_scope_map
 
-        tagged.sort_by { |r| r[:position] }.each do |remark|
-          next if @attached_spans.include?(remark[:position])
+        tagged.sort_by(&:position).each do |remark|
+          next if @attached_spans.include?(remark.position)
 
-          tag = remark[:tag]
+          tag = remark.tag
           target = nil
 
           # Find containing scope using pre-computed scope map (O(1))
           # Falls back to position-based lookup if scope map doesn't have the line
-          containing_scope = find_containing_scope_by_name(remark[:line])
-          containing_scope ||= find_containing_scope_position(remark[:line],
+          containing_scope = find_containing_scope_by_name(remark.line)
+          containing_scope ||= find_containing_scope_position(remark.line,
                                                               nodes_with_positions)
 
           # Check if this is an informal proposition tag (IP\d+)
           if tag.match?(/^IP\d+$/)
             scope = containing_scope
             if scope.nil?
-              scope = find_scope_by_source_text(remark[:line])
+              scope = find_scope_by_source_text(remark.line)
             end
             if scope && supports_informal_propositions?(scope)
               target = create_or_find_informal_proposition(scope, tag)
@@ -246,7 +165,7 @@ module Expressir
                 # Special handling for remarks inside WHERE clauses
                 if target.nil? && supports_where_rules?(containing_scope)
                   target = find_target_in_where_clause(containing_scope, tag,
-                                                       remark[:line])
+                                                       remark.line)
                 end
 
                 # Only fall back to schema prefix if NOT inside a function/rule/procedure
@@ -296,9 +215,8 @@ module Expressir
           end
 
           if target
-            add_remark(target, remark[:text], format: remark[:format],
-                                              tag: remark[:tag])
-            @attached_spans << remark[:position]
+            add_remark(target, remark.text, format: remark.format, tag: remark.tag)
+            @attached_spans << remark.position
           end
         end
       end
@@ -829,28 +747,26 @@ module Expressir
       end
 
       def attach_untagged_remarks(remarks, nodes_with_positions)
-        untagged = remarks.reject { |r| r[:tag] }
+        untagged = remarks.reject(&:tag)
         return unless untagged.any?
 
         untagged.each do |remark|
-          next if @attached_spans.include?(remark[:position])
+          next if @attached_spans.include?(remark.position)
 
-          if end_scope_line?(remark[:line])
+          if end_scope_line?(remark.line)
             matched_node = find_node_for_end_scope_remark(remark,
                                                           nodes_with_positions)
             if matched_node
-              add_remark(matched_node, remark[:text], format: remark[:format],
-                                                      tag: nil)
-              @attached_spans << remark[:position]
+              add_remark(matched_node, remark.text, format: remark.format, tag: nil)
+              @attached_spans << remark.position
               next
             end
           end
 
           matched_node = find_nearest_node(remark, nodes_with_positions)
           if matched_node
-            add_remark(matched_node, remark[:text], format: remark[:format],
-                                                    tag: nil)
-            @attached_spans << remark[:position]
+            add_remark(matched_node, remark.text, format: remark.format, tag: nil)
+            @attached_spans << remark.position
           end
         end
       end
@@ -868,7 +784,7 @@ module Expressir
       end
 
       def find_node_for_end_scope_remark(remark, nodes)
-        line_content = get_line_content(remark[:line])
+        line_content = get_line_content(remark.line)
 
         node_type = case line_content
                     when /END_SCHEMA/i then Model::Declarations::Schema
@@ -883,8 +799,8 @@ module Expressir
 
         matching_nodes = nodes.select do |n|
           n[:node].is_a?(node_type) &&
-            (n[:end_line] == remark[:line] ||
-             (n[:end_line] && n[:end_line] <= remark[:line] && n[:end_line] >= remark[:line] - 2))
+            (n[:end_line] == remark.line ||
+             (n[:end_line] && n[:end_line] <= remark.line && n[:end_line] >= remark.line - 2))
         end
 
         matching_nodes.first&.dig(:node) || find_node_by_type(nodes, node_type)
@@ -1038,7 +954,7 @@ module Expressir
       end
 
       def find_nearest_node(remark, nodes)
-        remark_line = remark[:line]
+        remark_line = remark.line
 
         # For tail remarks, prefer nodes that START on the same line
         # This handles cases like: "attr : STRING; -- tail remark"

data/lib/expressir/express/remark_scanner.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Expressir
+  module Express
+    # Scans EXPRESS source text and extracts all remarks (tail and embedded),
+    # correctly tracking embedded-remark nesting so that inline `--` inside
+    # a `(* ... *)` documentation block is NOT mistaken for a tail remark.
+    #
+    # Single responsibility: source string -> Array<Remark>. The scanner has
+    # no knowledge of the model; attaching remarks to model elements is the
+    # job of {RemarkAttacher}.
+    #
+    # Reference: ISO 10303-11 §7.1.6.
+    #   tail_remark     = '--' [ remark_tag ] { <any char except newline> } '\n'
+    #   embedded_remark = '(*' [ remark_tag ]
+    #                       { <any char> | nested embedded_remark } '*)'
+    #
+    # The scanner is a single-pass state machine with three states:
+    #   :top          — outside any remark; look for `--` or `(*`.
+    #   :in_tail       — inside a tail remark; look only for the next newline.
+    #   :in_embedded   — inside one or more nested embedded remarks; look
+    #                    for the matching `*)` or a nested `(*`, ignoring `--`.
+    class RemarkScanner
+      # Immutable value object describing a single extracted remark.
+      Remark = Struct.new(:position, :line, :text, :tag, :format, keyword_init: true) do
+        def tail?
+          format == TAIL_FORMAT
+        end
+
+        def embedded?
+          format == EMBEDDED_FORMAT
+        end
+
+        def tagged?
+          !tag.nil? && !tag.empty?
+        end
+      end
+
+      # Format discriminator values.
+      TAIL_FORMAT = "tail"
+      EMBEDDED_FORMAT = "embedded"
+
+      # Lexical markers recognised by EXPRESS.
+      TAIL_MARKER = "--"
+      EMBEDDED_OPEN = "(*"
+      EMBEDDED_CLOSE = "*)"
+
+      # Tag forms (see ISO 10303-11 §7.1.6.3 Remark tag).
+      # IP-style informal-proposition tags are recognised only in tail remarks
+      # (matching historic behaviour); embedded remarks use the quoted form.
+      INFORMAL_PROPOSITION_TAG = /\A(IP\d+):\s*(.*)\z/m
+      QUOTED_TAG = /\A"([^"]*)"\s*(.*)\z/m
+
+      # @param source [String] EXPRESS source text (any encoding).
+      def initialize(source)
+        @source = source
+        @source_bytes = source.b
+        @line_map = LineMap.new(@source_bytes)
+      end
+
+      # Returns an Array of {Remark}, in source order (by byte position).
+      # @return [Array<Remark>]
+      def scan
+        run_state_machine
+      end
+
+      alias call scan
+
+      private
+
+      def run_state_machine
+        remarks = []
+        src = @source_bytes
+        pos = 0
+
+        state = :top
+        embedded_depth = 0
+        tail_start = nil      # byte position of first content char after "--"
+        tail_line = nil
+        embedded_start = nil  # byte position of first content char after "(*"
+        embedded_line = nil
+
+        while pos < src.bytesize
+          case state
+          when :in_tail
+            nl = src.index("\n", pos)
+            if nl
+              emit_tail(remarks, tail_start, nl, tail_line)
+              pos = nl + 1
+              state = :top
+            else
+              emit_tail(remarks, tail_start, src.bytesize, tail_line)
+              return remarks
+            end
+          when :in_embedded
+            closing = src.index(EMBEDDED_CLOSE, pos)
+            nesting = src.index(EMBEDDED_OPEN, pos)
+            if nesting && (!closing || nesting < closing)
+              embedded_depth += 1
+              pos = nesting + EMBEDDED_OPEN.bytesize
+            elsif closing
+              embedded_depth -= 1
+              pos = closing + EMBEDDED_CLOSE.bytesize
+              if embedded_depth.zero?
+                emit_embedded(remarks, embedded_start, closing, embedded_line)
+                state = :top
+              end
+            else
+              # Unclosed embedded remark — terminate scan gracefully.
+              return remarks
+            end
+          else # :top
+            next_embed = src.index(EMBEDDED_OPEN, pos)
+            next_tail = src.index(TAIL_MARKER, pos)
+            if next_embed.nil? && next_tail.nil?
+              return remarks
+            elsif next_embed && (next_tail.nil? || next_embed <= next_tail)
+              embedded_start = next_embed + EMBEDDED_OPEN.bytesize
+              embedded_line = @line_map.line_number(next_embed)
+              embedded_depth = 1
+              state = :in_embedded
+              pos = next_embed + EMBEDDED_OPEN.bytesize
+            else
+              tail_start = next_tail + TAIL_MARKER.bytesize
+              tail_line = @line_map.line_number(next_tail)
+              state = :in_tail
+              pos = next_tail + TAIL_MARKER.bytesize
+            end
+          end
+        end
+
+        remarks
+      end
+
+      def emit_tail(remarks, content_start, content_end, line)
+        # Slice content from the original (UTF-8) source so the returned text
+        # preserves the source encoding. Byte offsets come from the BINARY copy.
+        raw = @source.byteslice(content_start...content_end)
+        tag, content = parse_tail(raw.strip)
+        remarks << Remark.new(
+          position: content_start - TAIL_MARKER.bytesize,
+          line: line,
+          text: content,
+          tag: tag,
+          format: TAIL_FORMAT,
+        )
+      end
+
+      def emit_embedded(remarks, content_start, content_end, line)
+        raw = @source.byteslice(content_start...content_end)
+        tag, content = parse_embedded(raw)
+        remarks << Remark.new(
+          position: content_start - EMBEDDED_OPEN.bytesize,
+          line: line,
+          text: content,
+          tag: tag,
+          format: EMBEDDED_FORMAT,
+        )
+      end
+
+      # Tail remarks support both quoted tags ("schema.entity" ...) and the
+      # shorthand IP-tag form (IP1: ...). Returns [tag, content]; for untagged
+      # remarks tag is nil and content is the stripped text.
+      def parse_tail(text)
+        match = text.match(INFORMAL_PROPOSITION_TAG) || text.match(QUOTED_TAG)
+        match ? match.captures : [nil, text]
+      end
+
+      # Embedded remarks only use the quoted tag form.
+      def parse_embedded(content)
+        stripped = content.strip
+        if (m = stripped.match(QUOTED_TAG))
+          [m[1], m[2]]
+        else
+          [nil, stripped]
+        end
+      end
+    end
+  end
+end

data/lib/expressir/express.rb

@@ -12,10 +12,12 @@ module Expressir
     autoload :Formatter, "#{__dir__}/express/formatter"
     autoload :Formatters, "#{__dir__}/express/formatters"
     autoload :HyperlinkFormatter, "#{__dir__}/express/hyperlink_formatter"
+    autoload :LineMap, "#{__dir__}/express/line_map"
     autoload :ModelVisitor, "#{__dir__}/express/model_visitor"
     autoload :Parser, "#{__dir__}/express/parser"
     autoload :PrettyFormatter, "#{__dir__}/express/pretty_formatter"
     autoload :RemarkAttacher, "#{__dir__}/express/remark_attacher"
+    autoload :RemarkScanner, "#{__dir__}/express/remark_scanner"
     autoload :ResolveReferencesModelVisitor,
              "#{__dir__}/express/resolve_references_model_visitor"
     autoload :SchemaHeadFormatter, "#{__dir__}/express/schema_head_formatter"

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "2.3.7".freeze
+  VERSION = "2.4.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: expressir
 version: !ruby/object:Gem::Version
-  version: 2.3.7
+  version: 2.4.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-05 00:00:00.000000000 Z
+date: 2026-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -398,10 +398,12 @@ files:
 - lib/expressir/express/formatters/statements_formatter.rb
 - lib/expressir/express/formatters/supertype_expressions_formatter.rb
 - lib/expressir/express/hyperlink_formatter.rb
+- lib/expressir/express/line_map.rb
 - lib/expressir/express/model_visitor.rb
 - lib/expressir/express/parser.rb
 - lib/expressir/express/pretty_formatter.rb
 - lib/expressir/express/remark_attacher.rb
+- lib/expressir/express/remark_scanner.rb
 - lib/expressir/express/resolve_references_model_visitor.rb
 - lib/expressir/express/schema_head_formatter.rb
 - lib/expressir/express/streaming_builder.rb