ibex 0.1.0 → 0.3.0

data/lib/ibex/frontend/rule_documentation.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Attaches immediately preceding lossless line-comment blocks to parsed rules.
+    class RuleDocumentation
+      EMPTY_SEGMENTS = Array.new(0).freeze #: Array[Segment]
+      private_constant :EMPTY_SEGMENTS
+
+      # @rbs (AST::Root | AST::Fragment node, SourceDocument document) -> (AST::Root | AST::Fragment)
+      def self.enrich(node, document)
+        new(document).enrich(node)
+      end
+
+      # @rbs (SourceDocument document) -> void
+      def initialize(document)
+        @segments_by_line = index_segments(document) #: Hash[Integer, Array[Segment]]
+        @inline_lines = document.tokens.filter_map do |token|
+          token.location.line if token.type == :inline
+        end #: Array[Integer]
+      end
+
+      # @rbs (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+      def enrich(node)
+        rules = node.rules.map { |rule| documented_rule(rule) }
+        if node.is_a?(AST::Root)
+          AST::Root.new(
+            class_name: node.class_name, superclass: node.superclass, declarations: node.declarations,
+            rules: rules, user_code: node.user_code, loc: node.loc, extended: node.extended, cst: node.cst,
+            extended_loc: node.extended_loc
+          )
+        else
+          AST::Fragment.new(declarations: node.declarations, rules: rules, loc: node.loc)
+        end
+      end
+
+      private
+
+      # @rbs (AST::Rule rule) -> AST::Rule
+      def documented_rule(rule)
+        AST::Rule.new(
+          lhs: rule.lhs, parameters: rule.parameters, alternatives: rule.alternatives, loc: rule.loc,
+          documentation: documentation_before(documentation_anchor(rule)), inline: rule.inline
+        )
+      end
+
+      # @rbs (AST::Rule rule) -> Integer
+      def documentation_anchor(rule)
+        return rule.loc.line unless rule.inline
+
+        @inline_lines.reverse_each.find { |line| line <= rule.loc.line } || rule.loc.line
+      end
+
+      # @rbs (Integer rule_line) -> String?
+      def documentation_before(rule_line)
+        lines = [] #: Array[String]
+        line = rule_line - 1
+        while line.positive?
+          content = documentation_line(line)
+          break unless content
+
+          lines.unshift(content)
+          line -= 1
+        end
+        lines.empty? ? nil : lines.join("\n").freeze
+      end
+
+      # @rbs (Integer line) -> String?
+      def documentation_line(line)
+        segments = @segments_by_line.fetch(line, EMPTY_SEGMENTS)
+        comments = segments.select { |segment| segment.kind == :line_comment }
+        return unless comments.length == 1
+
+        comment = comments.fetch(0)
+        return unless comment.text.start_with?("##")
+        return unless segments.all? { |segment| documentation_line_segment?(segment, comment) }
+
+        comment.text.delete_prefix("##").delete_prefix(" ")
+      end
+
+      # @rbs (Segment segment, Segment comment) -> bool
+      def documentation_line_segment?(segment, comment)
+        segment.equal?(comment) || %i[whitespace newline].include?(segment.kind)
+      end
+
+      # @rbs (SourceDocument document) -> Hash[Integer, Array[Segment]]
+      def index_segments(document)
+        indexed = Hash.new { |hash, line| hash[line] = [] } #: Hash[Integer, Array[Segment]]
+        document.cst.each do |segment|
+          covered_lines(segment).each { |line| indexed[line] << segment }
+        end
+        indexed
+      end
+
+      # @rbs (Segment segment) -> Array[Integer]
+      def covered_lines(segment)
+        first = segment.span.start.line
+        return [first] if segment.kind == :newline
+
+        (first..segment.span.finish.line).to_a
+      end
+    end
+  end
+end

data/lib/ibex/frontend/source_cursor.rb

@@ -27,11 +27,12 @@ module Ibex
       :type, #: Symbol
       :value, #: token_value
       :location, #: Location
+      :span, #: SourceSpan?
       keyword_init: true
     )
 
     class Token
-      # @rbs () -> Hash[Symbol, untyped]
+      # @rbs () -> Hash[Symbol, Object?]
       def to_h
         { type: type, value: value, location: location.to_h }
       end
@@ -42,31 +43,53 @@ module Ibex
       attr_reader :source #: String
       attr_reader :file #: String
       attr_reader :index #: Integer
+      attr_reader :byte_offset #: Integer
       attr_reader :line #: Integer
       attr_reader :column #: Integer
 
       # @rbs (String source, String file) -> void
       def initialize(source, file)
-        @source = source
-        @file = file
+        @source = SourceEncoding.validated_utf8(source, file)
+        @file = SourceEncoding.validated_file(file)
+        @ascii_only = @source.ascii_only?
+        @character_length = @ascii_only ? @source.bytesize : @source.length
         @index = 0
+        @byte_offset = 0
         @line = 1
         @column = 1
       end
 
       # @rbs () -> bool
       def eof?
-        @index >= @source.length
+        @index >= @character_length
       end
 
       # @rbs (?Integer offset) -> String?
       def peek(offset = 0)
-        @source[@index + offset]
+        character_index = @index + offset
+        if @ascii_only
+          return if character_index >= @character_length
+
+          return @source.byteslice(character_index, 1)
+        end
+
+        character_index += @character_length if character_index.negative?
+        return if character_index.negative? || character_index >= @character_length
+
+        start_byte = if character_index == @index
+                       @byte_offset
+                     elsif character_index == @index + 1
+                       @byte_offset + utf8_character_width(@byte_offset)
+                     else
+                       byte_offset_for_character(character_index)
+                     end
+        @source.byteslice(start_byte, utf8_character_width(start_byte))
       end
 
       # @rbs () -> String
       def rest
-        @source[@index..] || ""
+        @source.byteslice(@byte_offset, @source.bytesize - @byte_offset) ||
+          String.new(encoding: Encoding::UTF_8)
       end
 
       # @rbs () -> Location
@@ -74,14 +97,34 @@ module Ibex
         Location.new(file: @file, line: @line, column: @column)
       end
 
+      # @rbs () -> SourcePosition
+      def position
+        SourcePosition.new(byte_offset: @byte_offset, line: @line, column: @column)
+      end
+
+      # @rbs (SourcePosition start) -> SourceSpan
+      def span_from(start)
+        SourceSpan.new(file: @file, start: start, finish: position)
+      end
+
       # @rbs (?Integer count) -> void
       def advance(count = 1)
+        return advance_utf8(count) unless @ascii_only
+
+        advance_ascii(count)
+      end
+
+      private
+
+      # @rbs (Integer count) -> void
+      def advance_ascii(count)
         count.times do
-          character = @source[@index]
-          break unless character
+          break if @index >= @character_length
 
+          newline = @source.getbyte(@index) == 10
           @index += 1
-          if character == "\n"
+          @byte_offset = @index
+          if newline
             @line += 1
             @column = 1
           else
@@ -89,6 +132,87 @@ module Ibex
           end
         end
       end
+
+      # @rbs (Integer count) -> void
+      def advance_utf8(count)
+        count.times do
+          break if @index >= @character_length
+
+          leading_byte = @source.getbyte(@byte_offset)
+          raise Ibex::Error, "#{@file}: input must be valid UTF-8" unless leading_byte
+
+          @index += 1
+          @byte_offset += if leading_byte < 0x80
+                            1
+                          elsif leading_byte < 0xE0
+                            2
+                          elsif leading_byte < 0xF0
+                            3
+                          else
+                            4
+                          end
+          if leading_byte == 10
+            @line += 1
+            @column = 1
+          else
+            @column += 1
+          end
+        end
+      end
+
+      # @rbs (Integer character_index) -> Integer
+      def byte_offset_for_character(character_index)
+        start_distance = character_index
+        current_distance = (character_index - @index).abs
+        finish_distance = @character_length - character_index
+
+        if current_distance <= start_distance && current_distance <= finish_distance
+          walk_to_character(@byte_offset, @index, character_index)
+        elsif start_distance <= finish_distance
+          walk_to_character(0, 0, character_index)
+        else
+          walk_to_character(@source.bytesize, @character_length, character_index)
+        end
+      end
+
+      # @rbs (Integer byte_offset, Integer from_index, Integer to_index) -> Integer
+      def walk_to_character(byte_offset, from_index, to_index)
+        while from_index < to_index
+          byte_offset += utf8_character_width(byte_offset)
+          from_index += 1
+        end
+        while from_index > to_index
+          byte_offset = previous_character_byte_offset(byte_offset)
+          from_index -= 1
+        end
+        byte_offset
+      end
+
+      # @rbs (Integer byte_offset) -> Integer
+      def previous_character_byte_offset(byte_offset)
+        byte_offset -= 1
+        while (byte = @source.getbyte(byte_offset)) && continuation_byte?(byte)
+          byte_offset -= 1
+        end
+        byte_offset
+      end
+
+      # @rbs (Integer byte_offset) -> Integer
+      def utf8_character_width(byte_offset)
+        leading_byte = @source.getbyte(byte_offset)
+        raise Ibex::Error, "#{@file}: input must be valid UTF-8" unless leading_byte
+
+        return 1 if leading_byte < 0x80
+        return 2 if leading_byte < 0xE0
+        return 3 if leading_byte < 0xF0
+
+        4
+      end
+
+      # @rbs (Integer byte) -> bool
+      def continuation_byte?(byte)
+        byte >= 0x80 && byte < 0xC0
+      end
     end
   end
 end

data/lib/ibex/frontend/source_document.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # An immutable lexical unit or trivia slice in a concrete syntax tree.
+    class Segment
+      TRIVIA_KINDS = %i[whitespace newline line_comment block_comment].freeze #: Array[Symbol]
+      OPAQUE_KINDS = %i[action user_code_body].freeze #: Array[Symbol]
+
+      attr_reader :kind #: Symbol
+      attr_reader :span #: SourceSpan
+      attr_reader :text #: String
+      attr_reader :token_type #: Symbol?
+      attr_reader :token_index #: Integer?
+
+      # @rbs (kind: Symbol, span: SourceSpan, text: String, ?token_type: Symbol?,
+      #   ?token_index: Integer?) -> void
+      def initialize(kind:, span:, text:, token_type: nil, token_index: nil)
+        raise ArgumentError, "segment text does not match its span" unless text.bytesize == span.length
+
+        @kind = kind
+        @span = span
+        @text = text.dup.freeze
+        @token_type = token_type
+        @token_index = token_index
+        freeze
+      end
+
+      # @rbs () -> bool
+      def trivia?
+        TRIVIA_KINDS.include?(kind)
+      end
+
+      # @rbs () -> bool
+      def opaque?
+        OPAQUE_KINDS.include?(kind)
+      end
+
+      # @rbs () -> bool
+      def token?
+        !token_index.nil?
+      end
+
+      # @rbs () -> String
+      def render
+        text
+      end
+    end
+
+    module CST
+      # The immutable lexical root of a grammar source document.
+      class Document
+        # @rbs! include Enumerable[Segment]
+        # @rbs skip
+        include Enumerable
+
+        attr_reader :segments #: Array[Segment]
+
+        # @rbs (Array[Segment] segments) -> void
+        def initialize(segments)
+          validate_contiguous(segments)
+          @segments = segments.dup.freeze
+          freeze
+        end
+
+        # @rbs!
+        #   def each: () -> Enumerator[Segment, Document]
+        #           | () { (Segment) -> void } -> Document
+        # @rbs skip
+        def each(&block)
+          return enum_for(:each) unless block
+
+          segments.each(&block)
+          self
+        end
+
+        # @rbs () -> String
+        def render
+          segments.map(&:render).join
+        end
+
+        private
+
+        # @rbs (Array[Segment] segments) -> void
+        def validate_contiguous(segments)
+          segments.each_cons(2) do |left, right|
+            next unless left && right
+            next if left.span.end_byte == right.span.start_byte
+
+            raise ArgumentError, "concrete syntax tree segments must be contiguous"
+          end
+        end
+      end
+    end
+
+    # Lossless grammar source plus its semantic parse and lexical concrete syntax tree.
+    class SourceDocument
+      attr_reader :source #: String
+      attr_reader :file #: String
+      attr_reader :tokens #: Array[Token]
+      attr_reader :cst #: CST::Document
+      attr_reader :ast #: AST::Root | AST::Fragment | nil
+
+      # @rbs (source: String, file: String, tokens: Array[Token], cst: CST::Document,
+      #   ?ast: AST::Root | AST::Fragment | nil) -> void
+      def initialize(source:, file:, tokens:, cst:, ast: nil)
+        @source = SourceEncoding.validated_utf8(source, file)
+        @file = file.dup.freeze
+        @tokens = tokens.dup.freeze
+        @cst = cst
+        @ast = ast
+        @line_starts = build_line_starts.freeze #: Array[Integer]
+        validate_render
+        freeze
+      end
+
+      # @rbs () -> String
+      def render
+        cst.render
+      end
+
+      # @rbs (SourceSpan span) -> String
+      def slice(span)
+        validate_span(span)
+        source.byteslice(span.start_byte, span.length) || ""
+      end
+
+      # @rbs (Integer byte_offset) -> SourcePosition
+      def position_at(byte_offset)
+        validate_byte_offset(byte_offset)
+        prefix = source.byteslice(0, byte_offset) || ""
+        raise ArgumentError, "byte offset must be on a UTF-8 character boundary" unless prefix.valid_encoding?
+
+        line_index = (@line_starts.bsearch_index { |start| start > byte_offset } || @line_starts.length) - 1
+        line_start = @line_starts.fetch(line_index)
+        line_prefix = source.byteslice(line_start, byte_offset - line_start) || ""
+        SourcePosition.new(byte_offset: byte_offset, line: line_index + 1, column: line_prefix.length + 1)
+      end
+
+      # @rbs (Integer line, Integer column) -> Integer
+      def byte_offset_at(line, column)
+        raise ArgumentError, "line and column must be positive" unless line.positive? && column.positive?
+
+        line_start = @line_starts[line - 1]
+        raise ArgumentError, "line is outside the source" unless line_start
+
+        line_end = @line_starts[line] || source.bytesize
+        line_text = source.byteslice(line_start, line_end - line_start) || ""
+        line_text = line_text.delete_suffix("\n")
+        character_count = column - 1
+        raise ArgumentError, "column is outside the source line" if character_count > line_text.length
+
+        line_start + line_text.each_char.take(character_count).join.bytesize
+      end
+
+      # @rbs (Integer start_byte, Integer end_byte) -> SourceSpan
+      def span(start_byte, end_byte)
+        SourceSpan.new(file: file, start: position_at(start_byte), finish: position_at(end_byte))
+      end
+
+      # @rbs (Segment segment) -> Token?
+      def token_for(segment)
+        index = segment.token_index
+        index && tokens[index]
+      end
+
+      # @rbs (AST::Root | AST::Fragment ast) -> SourceDocument
+      def with_ast(ast)
+        self.class.new(source: source, file: file, tokens: tokens, cst: cst, ast: ast)
+      end
+
+      private
+
+      # @rbs () -> Array[Integer]
+      def build_line_starts
+        starts = [0]
+        offset = 0
+        binary_source = source.b
+        while (newline = binary_source.index("\n", offset))
+          offset = newline + 1
+          starts << offset
+        end
+        starts
+      end
+
+      # @rbs () -> void
+      def validate_render
+        return if full_source_coverage? && segment_slices_match? && render == source
+
+        raise ArgumentError, "concrete syntax tree does not reproduce its source"
+      end
+
+      # @rbs () -> bool
+      def full_source_coverage?
+        segments = cst.segments
+        return source.empty? if segments.empty?
+
+        first = segments.first
+        last = segments.last
+        return false unless first && last
+
+        first.span.start_byte.zero? && last.span.end_byte == source.bytesize
+      end
+
+      # @rbs () -> bool
+      def segment_slices_match?
+        cst.segments.all? do |segment|
+          segment.span.file == file &&
+            source.byteslice(segment.span.start_byte, segment.span.length) == segment.text
+        end
+      end
+
+      # @rbs (Integer byte_offset) -> void
+      def validate_byte_offset(byte_offset)
+        return if byte_offset.between?(0, source.bytesize)
+
+        raise ArgumentError, "byte offset is outside the source"
+      end
+
+      # @rbs (SourceSpan span) -> void
+      def validate_span(span)
+        raise ArgumentError, "span belongs to another source" unless span.file == file
+
+        validate_byte_offset(span.start_byte)
+        validate_byte_offset(span.end_byte)
+      end
+    end
+  end
+end

data/lib/ibex/frontend/source_loader.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require_relative "../generation_input"
+
+module Ibex
+  module Frontend
+    # Resolves canonical grammar paths and reads either open-buffer overlays or disk.
+    class SourceLoader
+      # @rbs @overlays: Hash[String, String]
+      # @rbs @aliases: Hash[String, String]
+      # @rbs @read_records: Hash[String, GenerationInput]
+      # @rbs @record_reads: bool
+      # @rbs @access_paths: Hash[String, Array[String]]
+
+      # @rbs () -> Array[GenerationInput]
+      def read_records
+        @read_records.values
+      end
+
+      # Associate the lexical path used by a resolver with its canonical source.
+      # @rbs (String canonical, String access_path) -> void
+      def record_access(canonical, access_path)
+        paths = (@access_paths[canonical] ||= [])
+        expanded = File.expand_path(access_path)
+        paths << expanded unless paths.include?(expanded)
+        @read_records[canonical]&.add_access_path(expanded)
+      end
+
+      # @rbs (?overlays: Hash[String, String], ?record_reads: bool) -> void
+      def initialize(overlays: {}, record_reads: false)
+        @overlays = {} #: Hash[String, String]
+        @aliases = {} #: Hash[String, String]
+        @read_records = {} #: Hash[String, GenerationInput]
+        @record_reads = record_reads
+        @access_paths = {} #: Hash[String, Array[String]]
+        overlays.each { |path, source| set_overlay(path, source) }
+      end
+
+      # Canonicalize an existing file, or a missing path explicitly allowed by an open overlay.
+      # @rbs (String path, ?base: String, ?allow_missing: bool) -> String
+      def canonical_path(path, base: Dir.pwd, allow_missing: false)
+        expanded = File.expand_path(path, base)
+        aliased = @aliases[expanded]
+        return aliased if aliased
+
+        begin
+          File.realpath(expanded)
+        rescue Errno::ENOENT
+          canonical = canonical_missing_path(expanded)
+          return canonical if allow_missing || @overlays.key?(canonical)
+
+          raise
+        end
+      end
+
+      # @rbs (String path) -> String
+      def read(path)
+        canonical = canonical_path(path, allow_missing: true)
+        overlay = @overlays[canonical]
+        source = overlay || File.binread(canonical)
+        if @record_reads
+          record = GenerationInput.new(canonical, source)
+          (@access_paths[canonical] || []).each { |access| record.add_access_path(access) }
+          @read_records[canonical] = record
+        end
+
+        source
+      end
+
+      # @rbs (String path) -> bool
+      def file?(path)
+        canonical = canonical_path(path, allow_missing: true)
+        @overlays.key?(canonical) || File.file?(canonical)
+      rescue SystemCallError
+        false
+      end
+
+      # @rbs (String path) -> bool
+      def overlay?(path)
+        canonical = canonical_path(path, allow_missing: true)
+        @overlays.key?(canonical)
+      rescue SystemCallError
+        false
+      end
+
+      # Install or replace an editor buffer without writing it to disk.
+      # @rbs (String path, String source) -> String
+      def set_overlay(path, source)
+        expanded = File.expand_path(path)
+        canonical = canonical_path(expanded, allow_missing: true)
+        raise ArgumentError, "overlay path must not be a directory" if File.directory?(canonical)
+
+        @aliases[expanded] = canonical
+        @overlays[canonical] = source.dup.freeze
+        canonical
+      end
+
+      # Remove an editor buffer and return its canonical path.
+      # @rbs (String path) -> String
+      def delete_overlay(path)
+        expanded = File.expand_path(path)
+        canonical = @aliases.delete(expanded) || canonical_path(expanded, allow_missing: true)
+        @aliases.delete_if { |_alias_path, target| target == canonical }
+        @overlays.delete(canonical)
+        canonical
+      end
+
+      # Read disk while deliberately bypassing an open overlay.
+      # @rbs (String path) -> String
+      def disk_source(path)
+        File.binread(disk_path(path))
+      end
+
+      # @rbs (String path) -> bool
+      def disk_file?(path)
+        File.file?(disk_path(path))
+      rescue SystemCallError
+        false
+      end
+
+      # Resolve the current disk target without consulting overlay aliases.
+      # @rbs (String path) -> String
+      def disk_path(path)
+        File.realpath(File.expand_path(path))
+      end
+
+      private
+
+      # Resolve the nearest existing ancestor so symlink escapes remain visible for new files.
+      # @rbs (String expanded) -> String
+      def canonical_missing_path(expanded)
+        raise Errno::ENOENT, expanded if File.symlink?(expanded)
+
+        cursor = expanded
+        suffix = [] #: Array[String]
+        loop do
+          parent = File.dirname(cursor)
+          raise Errno::ENOENT, expanded if parent == cursor
+
+          suffix.unshift(File.basename(cursor))
+          cursor = parent
+          break if File.exist?(cursor) || File.symlink?(cursor)
+        end
+        raise Errno::ENOTDIR, cursor unless File.directory?(File.realpath(cursor))
+
+        File.join(File.realpath(cursor), *suffix)
+      end
+    end
+  end
+end