ibex 0.1.0 → 0.2.0

data/lib/ibex/frontend/source_cursor.rb

@@ -27,6 +27,7 @@ module Ibex
       :type, #: Symbol
       :value, #: token_value
       :location, #: Location
+      :span, #: SourceSpan?
       keyword_init: true
     )
 
@@ -42,31 +43,53 @@ module Ibex
       attr_reader :source #: String
       attr_reader :file #: String
       attr_reader :index #: Integer
+      attr_reader :byte_offset #: Integer
       attr_reader :line #: Integer
       attr_reader :column #: Integer
 
       # @rbs (String source, String file) -> void
       def initialize(source, file)
-        @source = source
-        @file = file
+        @source = SourceEncoding.validated_utf8(source, file)
+        @file = file.dup.freeze
+        @ascii_only = @source.ascii_only?
+        @character_length = @ascii_only ? @source.bytesize : @source.length
         @index = 0
+        @byte_offset = 0
         @line = 1
         @column = 1
       end
 
       # @rbs () -> bool
       def eof?
-        @index >= @source.length
+        @index >= @character_length
       end
 
       # @rbs (?Integer offset) -> String?
       def peek(offset = 0)
-        @source[@index + offset]
+        character_index = @index + offset
+        if @ascii_only
+          return if character_index >= @character_length
+
+          return @source.byteslice(character_index, 1)
+        end
+
+        character_index += @character_length if character_index.negative?
+        return if character_index.negative? || character_index >= @character_length
+
+        start_byte = if character_index == @index
+                       @byte_offset
+                     elsif character_index == @index + 1
+                       @byte_offset + utf8_character_width(@byte_offset)
+                     else
+                       byte_offset_for_character(character_index)
+                     end
+        @source.byteslice(start_byte, utf8_character_width(start_byte))
       end
 
       # @rbs () -> String
       def rest
-        @source[@index..] || ""
+        @source.byteslice(@byte_offset, @source.bytesize - @byte_offset) ||
+          String.new(encoding: Encoding::UTF_8)
       end
 
       # @rbs () -> Location
@@ -74,14 +97,34 @@ module Ibex
         Location.new(file: @file, line: @line, column: @column)
       end
 
+      # @rbs () -> SourcePosition
+      def position
+        SourcePosition.new(byte_offset: @byte_offset, line: @line, column: @column)
+      end
+
+      # @rbs (SourcePosition start) -> SourceSpan
+      def span_from(start)
+        SourceSpan.new(file: @file, start: start, finish: position)
+      end
+
       # @rbs (?Integer count) -> void
       def advance(count = 1)
+        return advance_utf8(count) unless @ascii_only
+
+        advance_ascii(count)
+      end
+
+      private
+
+      # @rbs (Integer count) -> void
+      def advance_ascii(count)
         count.times do
-          character = @source[@index]
-          break unless character
+          break if @index >= @character_length
 
+          newline = @source.getbyte(@index) == 10
           @index += 1
-          if character == "\n"
+          @byte_offset = @index
+          if newline
             @line += 1
             @column = 1
           else
@@ -89,6 +132,87 @@ module Ibex
           end
         end
       end
+
+      # @rbs (Integer count) -> void
+      def advance_utf8(count)
+        count.times do
+          break if @index >= @character_length
+
+          leading_byte = @source.getbyte(@byte_offset)
+          raise Ibex::Error, "#{@file}: input must be valid UTF-8" unless leading_byte
+
+          @index += 1
+          @byte_offset += if leading_byte < 0x80
+                            1
+                          elsif leading_byte < 0xE0
+                            2
+                          elsif leading_byte < 0xF0
+                            3
+                          else
+                            4
+                          end
+          if leading_byte == 10
+            @line += 1
+            @column = 1
+          else
+            @column += 1
+          end
+        end
+      end
+
+      # @rbs (Integer character_index) -> Integer
+      def byte_offset_for_character(character_index)
+        start_distance = character_index
+        current_distance = (character_index - @index).abs
+        finish_distance = @character_length - character_index
+
+        if current_distance <= start_distance && current_distance <= finish_distance
+          walk_to_character(@byte_offset, @index, character_index)
+        elsif start_distance <= finish_distance
+          walk_to_character(0, 0, character_index)
+        else
+          walk_to_character(@source.bytesize, @character_length, character_index)
+        end
+      end
+
+      # @rbs (Integer byte_offset, Integer from_index, Integer to_index) -> Integer
+      def walk_to_character(byte_offset, from_index, to_index)
+        while from_index < to_index
+          byte_offset += utf8_character_width(byte_offset)
+          from_index += 1
+        end
+        while from_index > to_index
+          byte_offset = previous_character_byte_offset(byte_offset)
+          from_index -= 1
+        end
+        byte_offset
+      end
+
+      # @rbs (Integer byte_offset) -> Integer
+      def previous_character_byte_offset(byte_offset)
+        byte_offset -= 1
+        while (byte = @source.getbyte(byte_offset)) && continuation_byte?(byte)
+          byte_offset -= 1
+        end
+        byte_offset
+      end
+
+      # @rbs (Integer byte_offset) -> Integer
+      def utf8_character_width(byte_offset)
+        leading_byte = @source.getbyte(byte_offset)
+        raise Ibex::Error, "#{@file}: input must be valid UTF-8" unless leading_byte
+
+        return 1 if leading_byte < 0x80
+        return 2 if leading_byte < 0xE0
+        return 3 if leading_byte < 0xF0
+
+        4
+      end
+
+      # @rbs (Integer byte) -> bool
+      def continuation_byte?(byte)
+        byte >= 0x80 && byte < 0xC0
+      end
     end
   end
 end

data/lib/ibex/frontend/source_document.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # An immutable lexical unit or trivia slice in a concrete syntax tree.
+    class Segment
+      TRIVIA_KINDS = %i[whitespace newline line_comment block_comment].freeze #: Array[Symbol]
+      OPAQUE_KINDS = %i[action user_code_body].freeze #: Array[Symbol]
+
+      attr_reader :kind #: Symbol
+      attr_reader :span #: SourceSpan
+      attr_reader :text #: String
+      attr_reader :token_type #: Symbol?
+      attr_reader :token_index #: Integer?
+
+      # @rbs (kind: Symbol, span: SourceSpan, text: String, ?token_type: Symbol?,
+      #   ?token_index: Integer?) -> void
+      def initialize(kind:, span:, text:, token_type: nil, token_index: nil)
+        raise ArgumentError, "segment text does not match its span" unless text.bytesize == span.length
+
+        @kind = kind
+        @span = span
+        @text = text.dup.freeze
+        @token_type = token_type
+        @token_index = token_index
+        freeze
+      end
+
+      # @rbs () -> bool
+      def trivia?
+        TRIVIA_KINDS.include?(kind)
+      end
+
+      # @rbs () -> bool
+      def opaque?
+        OPAQUE_KINDS.include?(kind)
+      end
+
+      # @rbs () -> bool
+      def token?
+        !token_index.nil?
+      end
+
+      # @rbs () -> String
+      def render
+        text
+      end
+    end
+
+    module CST
+      # The immutable lexical root of a grammar source document.
+      class Document
+        # @rbs! include Enumerable[Segment]
+        # @rbs skip
+        include Enumerable
+
+        attr_reader :segments #: Array[Segment]
+
+        # @rbs (Array[Segment] segments) -> void
+        def initialize(segments)
+          validate_contiguous(segments)
+          @segments = segments.dup.freeze
+          freeze
+        end
+
+        # @rbs!
+        #   def each: () -> Enumerator[Segment, Document]
+        #           | () { (Segment) -> void } -> Document
+        # @rbs skip
+        def each(&block)
+          return enum_for(:each) unless block
+
+          segments.each(&block)
+          self
+        end
+
+        # @rbs () -> String
+        def render
+          segments.map(&:render).join
+        end
+
+        private
+
+        # @rbs (Array[Segment] segments) -> void
+        def validate_contiguous(segments)
+          segments.each_cons(2) do |left, right|
+            next unless left && right
+            next if left.span.end_byte == right.span.start_byte
+
+            raise ArgumentError, "concrete syntax tree segments must be contiguous"
+          end
+        end
+      end
+    end
+
+    # Lossless grammar source plus its semantic parse and lexical concrete syntax tree.
+    class SourceDocument
+      attr_reader :source #: String
+      attr_reader :file #: String
+      attr_reader :tokens #: Array[Token]
+      attr_reader :cst #: CST::Document
+      attr_reader :ast #: AST::Root | AST::Fragment | nil
+
+      # @rbs (source: String, file: String, tokens: Array[Token], cst: CST::Document,
+      #   ?ast: AST::Root | AST::Fragment | nil) -> void
+      def initialize(source:, file:, tokens:, cst:, ast: nil)
+        @source = SourceEncoding.validated_utf8(source, file)
+        @file = file.dup.freeze
+        @tokens = tokens.dup.freeze
+        @cst = cst
+        @ast = ast
+        @line_starts = build_line_starts.freeze #: Array[Integer]
+        validate_render
+        freeze
+      end
+
+      # @rbs () -> String
+      def render
+        cst.render
+      end
+
+      # @rbs (SourceSpan span) -> String
+      def slice(span)
+        validate_span(span)
+        source.byteslice(span.start_byte, span.length) || ""
+      end
+
+      # @rbs (Integer byte_offset) -> SourcePosition
+      def position_at(byte_offset)
+        validate_byte_offset(byte_offset)
+        prefix = source.byteslice(0, byte_offset) || ""
+        raise ArgumentError, "byte offset must be on a UTF-8 character boundary" unless prefix.valid_encoding?
+
+        line_index = (@line_starts.bsearch_index { |start| start > byte_offset } || @line_starts.length) - 1
+        line_start = @line_starts.fetch(line_index)
+        line_prefix = source.byteslice(line_start, byte_offset - line_start) || ""
+        SourcePosition.new(byte_offset: byte_offset, line: line_index + 1, column: line_prefix.length + 1)
+      end
+
+      # @rbs (Integer line, Integer column) -> Integer
+      def byte_offset_at(line, column)
+        raise ArgumentError, "line and column must be positive" unless line.positive? && column.positive?
+
+        line_start = @line_starts[line - 1]
+        raise ArgumentError, "line is outside the source" unless line_start
+
+        line_end = @line_starts[line] || source.bytesize
+        line_text = source.byteslice(line_start, line_end - line_start) || ""
+        line_text = line_text.delete_suffix("\n")
+        character_count = column - 1
+        raise ArgumentError, "column is outside the source line" if character_count > line_text.length
+
+        line_start + line_text.each_char.take(character_count).join.bytesize
+      end
+
+      # @rbs (Integer start_byte, Integer end_byte) -> SourceSpan
+      def span(start_byte, end_byte)
+        SourceSpan.new(file: file, start: position_at(start_byte), finish: position_at(end_byte))
+      end
+
+      # @rbs (Segment segment) -> Token?
+      def token_for(segment)
+        index = segment.token_index
+        index && tokens[index]
+      end
+
+      # @rbs (AST::Root | AST::Fragment ast) -> SourceDocument
+      def with_ast(ast)
+        self.class.new(source: source, file: file, tokens: tokens, cst: cst, ast: ast)
+      end
+
+      private
+
+      # @rbs () -> Array[Integer]
+      def build_line_starts
+        starts = [0]
+        offset = 0
+        binary_source = source.b
+        while (newline = binary_source.index("\n", offset))
+          offset = newline + 1
+          starts << offset
+        end
+        starts
+      end
+
+      # @rbs () -> void
+      def validate_render
+        return if full_source_coverage? && segment_slices_match? && render == source
+
+        raise ArgumentError, "concrete syntax tree does not reproduce its source"
+      end
+
+      # @rbs () -> bool
+      def full_source_coverage?
+        segments = cst.segments
+        return source.empty? if segments.empty?
+
+        first = segments.first
+        last = segments.last
+        return false unless first && last
+
+        first.span.start_byte.zero? && last.span.end_byte == source.bytesize
+      end
+
+      # @rbs () -> bool
+      def segment_slices_match?
+        cst.segments.all? do |segment|
+          segment.span.file == file &&
+            source.byteslice(segment.span.start_byte, segment.span.length) == segment.text
+        end
+      end
+
+      # @rbs (Integer byte_offset) -> void
+      def validate_byte_offset(byte_offset)
+        return if byte_offset.between?(0, source.bytesize)
+
+        raise ArgumentError, "byte offset is outside the source"
+      end
+
+      # @rbs (SourceSpan span) -> void
+      def validate_span(span)
+        raise ArgumentError, "span belongs to another source" unless span.file == file
+
+        validate_byte_offset(span.start_byte)
+        validate_byte_offset(span.end_byte)
+      end
+    end
+  end
+end

data/lib/ibex/frontend/source_loader.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require_relative "../generation_input"
+
+module Ibex
+  module Frontend
+    # Resolves canonical grammar paths and reads either open-buffer overlays or disk.
+    class SourceLoader
+      # @rbs @overlays: Hash[String, String]
+      # @rbs @aliases: Hash[String, String]
+      # @rbs @read_records: Hash[String, GenerationInput]
+      # @rbs @record_reads: bool
+      # @rbs @access_paths: Hash[String, Array[String]]
+
+      # @rbs () -> Array[GenerationInput]
+      def read_records
+        @read_records.values
+      end
+
+      # Associate the lexical path used by a resolver with its canonical source.
+      # @rbs (String canonical, String access_path) -> void
+      def record_access(canonical, access_path)
+        paths = (@access_paths[canonical] ||= [])
+        expanded = File.expand_path(access_path)
+        paths << expanded unless paths.include?(expanded)
+        @read_records[canonical]&.add_access_path(expanded)
+      end
+
+      # @rbs (?overlays: Hash[String, String], ?record_reads: bool) -> void
+      def initialize(overlays: {}, record_reads: false)
+        @overlays = {} #: Hash[String, String]
+        @aliases = {} #: Hash[String, String]
+        @read_records = {} #: Hash[String, GenerationInput]
+        @record_reads = record_reads
+        @access_paths = {} #: Hash[String, Array[String]]
+        overlays.each { |path, source| set_overlay(path, source) }
+      end
+
+      # Canonicalize an existing file, or a missing path explicitly allowed by an open overlay.
+      # @rbs (String path, ?base: String, ?allow_missing: bool) -> String
+      def canonical_path(path, base: Dir.pwd, allow_missing: false)
+        expanded = File.expand_path(path, base)
+        aliased = @aliases[expanded]
+        return aliased if aliased
+
+        begin
+          File.realpath(expanded)
+        rescue Errno::ENOENT
+          canonical = canonical_missing_path(expanded)
+          return canonical if allow_missing || @overlays.key?(canonical)
+
+          raise
+        end
+      end
+
+      # @rbs (String path) -> String
+      def read(path)
+        canonical = canonical_path(path, allow_missing: true)
+        overlay = @overlays[canonical]
+        source = overlay || File.binread(canonical)
+        if @record_reads
+          record = GenerationInput.new(canonical, source)
+          (@access_paths[canonical] || []).each { |access| record.add_access_path(access) }
+          @read_records[canonical] = record
+        end
+
+        source
+      end
+
+      # @rbs (String path) -> bool
+      def file?(path)
+        canonical = canonical_path(path, allow_missing: true)
+        @overlays.key?(canonical) || File.file?(canonical)
+      rescue SystemCallError
+        false
+      end
+
+      # @rbs (String path) -> bool
+      def overlay?(path)
+        canonical = canonical_path(path, allow_missing: true)
+        @overlays.key?(canonical)
+      rescue SystemCallError
+        false
+      end
+
+      # Install or replace an editor buffer without writing it to disk.
+      # @rbs (String path, String source) -> String
+      def set_overlay(path, source)
+        expanded = File.expand_path(path)
+        canonical = canonical_path(expanded, allow_missing: true)
+        raise ArgumentError, "overlay path must not be a directory" if File.directory?(canonical)
+
+        @aliases[expanded] = canonical
+        @overlays[canonical] = source.dup.freeze
+        canonical
+      end
+
+      # Remove an editor buffer and return its canonical path.
+      # @rbs (String path) -> String
+      def delete_overlay(path)
+        expanded = File.expand_path(path)
+        canonical = @aliases.delete(expanded) || canonical_path(expanded, allow_missing: true)
+        @aliases.delete_if { |_alias_path, target| target == canonical }
+        @overlays.delete(canonical)
+        canonical
+      end
+
+      # Read disk while deliberately bypassing an open overlay.
+      # @rbs (String path) -> String
+      def disk_source(path)
+        File.binread(disk_path(path))
+      end
+
+      # @rbs (String path) -> bool
+      def disk_file?(path)
+        File.file?(disk_path(path))
+      rescue SystemCallError
+        false
+      end
+
+      # Resolve the current disk target without consulting overlay aliases.
+      # @rbs (String path) -> String
+      def disk_path(path)
+        File.realpath(File.expand_path(path))
+      end
+
+      private
+
+      # Resolve the nearest existing ancestor so symlink escapes remain visible for new files.
+      # @rbs (String expanded) -> String
+      def canonical_missing_path(expanded)
+        raise Errno::ENOENT, expanded if File.symlink?(expanded)
+
+        cursor = expanded
+        suffix = [] #: Array[String]
+        loop do
+          parent = File.dirname(cursor)
+          raise Errno::ENOENT, expanded if parent == cursor
+
+          suffix.unshift(File.basename(cursor))
+          cursor = parent
+          break if File.exist?(cursor) || File.symlink?(cursor)
+        end
+        raise Errno::ENOTDIR, cursor unless File.directory?(File.realpath(cursor))
+
+        File.join(File.realpath(cursor), *suffix)
+      end
+    end
+  end
+end

data/lib/ibex/frontend/source_span.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Normalizes grammar source without changing its bytes.
+    module SourceEncoding
+      # @rbs (String source, String file) -> String
+      def self.validated_utf8(source, file)
+        validated = source.dup.force_encoding(Encoding::UTF_8)
+        raise Ibex::Error, "#{file}: input must be valid UTF-8" unless validated.valid_encoding?
+
+        validated.freeze
+      end
+    end
+
+    # An immutable position in grammar source.
+    class SourcePosition
+      attr_reader :byte_offset #: Integer
+      attr_reader :line #: Integer
+      attr_reader :column #: Integer
+
+      # @rbs (byte_offset: Integer, line: Integer, column: Integer) -> void
+      def initialize(byte_offset:, line:, column:)
+        raise ArgumentError, "byte_offset must be non-negative" if byte_offset.negative?
+        raise ArgumentError, "line and column must be positive" unless line.positive? && column.positive?
+
+        @byte_offset = byte_offset
+        @line = line
+        @column = column
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, Integer]
+      def to_h
+        { byte_offset: byte_offset, line: line, column: column }
+      end
+    end
+
+    # An immutable half-open byte span in one grammar source.
+    class SourceSpan
+      attr_reader :file #: String
+      attr_reader :start #: SourcePosition
+      attr_reader :finish #: SourcePosition
+
+      # @rbs (file: String, start: SourcePosition, finish: SourcePosition) -> void
+      def initialize(file:, start:, finish:)
+        raise ArgumentError, "span end precedes its start" if finish.byte_offset < start.byte_offset
+
+        @file = file.dup.freeze
+        @start = start
+        @finish = finish
+        freeze
+      end
+
+      # @rbs () -> Integer
+      def start_byte
+        start.byte_offset
+      end
+
+      # @rbs () -> Integer
+      def end_byte
+        finish.byte_offset
+      end
+
+      # @rbs () -> Integer
+      def length
+        end_byte - start_byte
+      end
+
+      # @rbs () -> bool
+      def empty?
+        length.zero?
+      end
+
+      # @rbs () -> Hash[Symbol, untyped]
+      def to_h
+        { file: file, start: start.to_h, end: finish.to_h }
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/declaration_document_state.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Root/fragment header and include-path transitions for declaration classification.
+      module DeclarationDocumentState
+        private
+
+        # @rbs (Token token) -> external_token
+        def class_keyword(token)
+          # @type self: DeclarationState
+          value = string_value(token)
+          if value == "fragment"
+            raise Ibex::Error, "#{token.location}: fragments require extended mode" unless @extended_mode
+
+            @fragment = true
+            @state = :declaration
+            return :FRAGMENT
+          end
+          return :IDENTIFIER unless value == "class"
+
+          @state = :class_name
+          :CLASS
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_include(type)
+          # @type self: DeclarationState
+          return unless @state == :include_path && type == :LITERAL
+
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (Token token, String value) -> void
+        def reject_fragment_pragma(token, value)
+          # @type self: DeclarationState
+          return unless @fragment && value == "pragma"
+
+          raise Ibex::Error, "#{token.location}: pragma declarations are not allowed in fragments"
+        end
+      end
+    end
+  end
+end