ibex 0.1.0 → 0.2.0

data/lib/ibex/lsp/symbol_index_builder.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Builds source occurrences and hover metadata from lossless frontend documents.
+    class SymbolIndexBuilder
+      include SymbolIndexSourceQueries
+      include SymbolIndexPrecedenceReferences
+
+      # @rbs (DocumentStore store, Array[String] files) -> void
+      def initialize(store, files)
+        @store = store
+        @files = files
+        @occurrences = [] #: Array[SymbolOccurrence]
+        @documents = {} #: Hash[String, Frontend::SourceDocument]
+        @global_kinds = {} #: Hash[String, Symbol]
+        @rule_data = {} #: Hash[String, Hash[Symbol, untyped]]
+        @terminal_data = Hash.new { |hash, key| hash[key] = {} } #: Hash[String, Hash[Symbol, untyped]]
+      end
+
+      # @rbs () -> [Array[SymbolOccurrence], Hash[String, Frontend::SourceDocument]]
+      def build
+        collect_documents
+        # Definitions from every closure file must exist before references are classified.
+        # rubocop:disable Style/CombinableLoops
+        @documents.each { |path, document| collect_definitions(path, document) }
+        @documents.each { |path, document| collect_references(path, document) }
+        # rubocop:enable Style/CombinableLoops
+        [@occurrences.freeze, @documents.freeze]
+      end
+
+      private
+
+      # @rbs () -> void
+      def collect_documents
+        @files.each do |path|
+          snapshot = @store.snapshot_for(path)
+          document = snapshot&.fetch(:document)
+          @documents[path] = document if document
+        end
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document) -> void
+      def collect_definitions(path, document)
+        ast = document.ast
+        return unless ast
+
+        declarations = ast.declarations
+        declarations.each_with_index do |declaration, index|
+          following = declarations[index + 1]&.loc || ast.rules.first&.loc
+          collect_declaration(path, document, declaration, following)
+        end
+        ast.rules.each { |rule| collect_rule_definition(path, document, rule) }
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::declaration declaration,
+      #   Frontend::Location? following) -> void
+      def collect_declaration(path, document, declaration, following)
+        tokens = tokens_between(document, declaration.loc, following)
+        case declaration
+        when Frontend::AST::Tokens
+          add_named_tokens(path, tokens.drop(1), declaration.names, :terminal, :definition)
+        when Frontend::AST::DisplayName
+          @terminal_data[declaration.name][:display] = declaration.value
+        when Frontend::AST::SemanticType
+          @terminal_data[declaration.name][:type] = declaration.value
+        when Frontend::AST::Precedence
+          collect_precedence(declaration)
+        when Frontend::AST::Include
+          collect_include(path, tokens, declaration)
+        end
+      end
+
+      # @rbs (Frontend::AST::Precedence declaration) -> void
+      def collect_precedence(declaration)
+        declaration.levels.each_with_index do |level, index|
+          level.symbols.each do |name|
+            @terminal_data[name][:precedence] = {
+              direction: declaration.direction, associativity: level.associativity, level: index
+            }
+          end
+        end
+      end
+
+      # @rbs (String path, Array[Frontend::Token] tokens, Frontend::AST::Include declaration) -> void
+      def collect_include(path, tokens, declaration)
+        token = tokens.find { |candidate| candidate.type == :literal }
+        return unless token&.span
+
+        target = canonical_include_target(declaration)
+        return unless target
+
+        add_occurrence(
+          name: declaration.path, kind: :include, role: :reference, key: [:include, target],
+          path: path, span: token.span, data: { target: target }
+        )
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::Rule rule) -> void
+      def collect_rule_definition(path, document, rule)
+        token = token_at(document, rule.loc, rule.lhs)
+        return unless token&.span
+
+        @global_kinds[rule.lhs] ||= :rule
+        signature = "#{rule.lhs}#{parameter_signature(rule.parameters)}"
+        signature = "%inline #{signature}" if rule.inline
+        data = {
+          signature: signature,
+          documentation: rule.documentation,
+          inline: rule.inline,
+          parameters: rule.parameters
+        }
+        @rule_data[rule.lhs] ||= data
+        add_occurrence(
+          name: rule.lhs, kind: :rule, role: :definition, key: global_key(rule.lhs),
+          path: path, span: token.span, data: data
+        )
+        collect_parameters(path, document, rule, token)
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::Rule rule,
+      #   Frontend::Token lhs_token) -> void
+      def collect_parameters(path, document, rule, lhs_token)
+        return if rule.parameters.empty?
+
+        rule_id = lhs_token.span&.start_byte || 0
+        rule.parameters.zip(parameter_tokens(document, lhs_token)).each do |parameter, token|
+          next unless token&.span
+
+          add_occurrence(
+            name: parameter, kind: :parameter, role: :definition,
+            key: parameter_key(path, rule_id, parameter), path: path, span: token.span,
+            data: { rule: rule.lhs }
+          )
+        end
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document) -> void
+      def collect_references(path, document)
+        ast = document.ast
+        return unless ast
+
+        ast.declarations.each_with_index do |declaration, index|
+          following = ast.declarations[index + 1]&.loc || ast.rules.first&.loc
+          collect_declaration_references(path, document, declaration, following)
+        end
+        ast.rules.each_with_index do |rule, index|
+          collect_rule_references(path, document, rule, ast.rules[index + 1]&.loc)
+        end
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::declaration declaration,
+      #   Frontend::Location? following) -> void
+      def collect_declaration_references(path, document, declaration, following)
+        names = declaration_reference_names(declaration)
+        return if names.empty?
+
+        remaining = tokens_between(document, declaration.loc, following).drop(1)
+        names.each do |name|
+          index = remaining.index { |token| token.value == name }
+          token = index && remaining.delete_at(index)
+          add_global_reference(path, token, name) if token
+        end
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::Rule rule,
+      #   Frontend::Location? following_rule) -> void
+      def collect_rule_references(path, document, rule, following_rule)
+        definition = token_at(document, rule.loc, rule.lhs)
+        rule_id = definition&.span&.start_byte || 0
+        rule.alternatives.each_with_index do |alternative, index|
+          alternative.items.each { |item| collect_item(path, document, item, rule, rule_id) }
+          following = rule.alternatives[index + 1]&.loc || following_rule
+          collect_alternative_precedence(path, document, alternative, following, rule, rule_id)
+        end
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::item item,
+      #   Frontend::AST::Rule rule, Integer rule_id) -> void
+      def collect_item(path, document, item, rule, rule_id)
+        case item
+        when Frontend::AST::SymbolReference, Frontend::AST::ParameterizedReference
+          collect_named_item(path, document, item, rule, rule_id)
+          item.arguments.each { |child| collect_item(path, document, child, rule, rule_id) } if
+            item.is_a?(Frontend::AST::ParameterizedReference)
+        when Frontend::AST::Optional, Frontend::AST::Star, Frontend::AST::Plus
+          collect_item(path, document, item.item, rule, rule_id)
+        when Frontend::AST::Group
+          item.alternatives.flatten.each { |child| collect_item(path, document, child, rule, rule_id) }
+        when Frontend::AST::SeparatedList
+          collect_item(path, document, item.item, rule, rule_id)
+          collect_item(path, document, item.separator, rule, rule_id)
+        end
+      end
+
+      # @rbs (String path, Frontend::SourceDocument document,
+      #   Frontend::AST::SymbolReference | Frontend::AST::ParameterizedReference item,
+      #   Frontend::AST::Rule rule, Integer rule_id) -> void
+      def collect_named_item(path, document, item, rule, rule_id)
+        token = token_at(document, item.loc, item.name)
+        return unless token&.span
+
+        add_rule_reference(path, token, item.name, rule, rule_id)
+      end
+
+      # @rbs (String path, Frontend::Token token, String name) -> void
+      def add_global_reference(path, token, name)
+        return unless token.span
+
+        kind = @global_kinds[name] || :terminal
+        data = symbol_data(kind, name)
+        add_occurrence(name: name, kind: kind, role: :reference, key: global_key(name),
+                       path: path, span: token.span, data: data)
+      end
+
+      # @rbs (Symbol kind, String name) -> Hash[Symbol, untyped]
+      def symbol_data(kind, name)
+        data = kind == :rule ? @rule_data[name] : @terminal_data[name]
+        empty = {} #: Hash[Symbol, untyped]
+        data || empty
+      end
+
+      # @rbs (String path, Array[Frontend::Token] tokens, Array[String] names, Symbol kind, Symbol role) -> void
+      def add_named_tokens(path, tokens, names, kind, role)
+        remaining = tokens.dup
+        names.each do |name|
+          index = remaining.index { |token| token.value == name }
+          token = index && remaining.delete_at(index)
+          next unless token&.span
+
+          @global_kinds[name] ||= kind
+          add_occurrence(name: name, kind: kind, role: role, key: global_key(name),
+                         path: path, span: token.span, data: @terminal_data[name])
+        end
+      end
+
+      # @rbs (**untyped attributes) -> void
+      def add_occurrence(**attributes)
+        @occurrences << SymbolOccurrence.new(**attributes)
+      end
+
+      # @rbs (String name) -> Array[untyped]
+      def global_key(name)
+        [:symbol, name]
+      end
+
+      # @rbs (String path, Integer rule_id, String name) -> Array[untyped]
+      def parameter_key(path, rule_id, name)
+        [:parameter, path, rule_id, name]
+      end
+
+      # @rbs (Array[String] parameters) -> String
+      def parameter_signature(parameters)
+        parameters.empty? ? "" : "(#{parameters.join(', ')})"
+      end
+
+      # @rbs (Frontend::AST::Include declaration) -> String?
+      def canonical_include_target(declaration)
+        candidate = File.expand_path(declaration.path, File.dirname(declaration.loc.file))
+        target = @store.loader.canonical_path(candidate, allow_missing: true)
+        @store.workspace.root_for(target) && target
+      rescue SystemCallError
+        nil
+      end
+    end
+  end
+end

data/lib/ibex/lsp/symbol_index_precedence_references.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Indexes rule-scoped item and alternative precedence references.
+    module SymbolIndexPrecedenceReferences
+      private
+
+      # @rbs (String path, Frontend::SourceDocument document, Frontend::AST::Alternative alternative,
+      #   Frontend::Location? following, Frontend::AST::Rule rule, Integer rule_id) -> void
+      def collect_alternative_precedence(path, document, alternative, following, rule, rule_id)
+        # @type self: SymbolIndexBuilder
+        name = alternative.precedence
+        return unless name
+
+        tokens = tokens_between(document, alternative.loc, following)
+        index = (0...(tokens.length - 1)).to_a.reverse.find do |candidate|
+          tokens.fetch(candidate).value == "=" && tokens.fetch(candidate + 1).value == name
+        end
+        token = index && tokens[index + 1]
+        add_rule_reference(path, token, name, rule, rule_id) if token
+      end
+
+      # @rbs (String path, Frontend::Token token, String name, Frontend::AST::Rule rule,
+      #   Integer rule_id) -> void
+      def add_rule_reference(path, token, name, rule, rule_id)
+        # @type self: SymbolIndexBuilder
+        return unless token.span
+
+        if rule.parameters.include?(name)
+          key = parameter_key(path, rule_id, name)
+          kind = :parameter
+          data = { rule: rule.lhs }
+        else
+          key = global_key(name)
+          kind = @global_kinds[name] || :rule
+          data = symbol_data(kind, name)
+        end
+        add_occurrence(name: name, kind: kind, role: :reference, key: key,
+                       path: path, span: token.span, data: data)
+      end
+    end
+  end
+end

data/lib/ibex/lsp/symbol_index_source_queries.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Locates lossless frontend tokens used by the symbol index.
+    module SymbolIndexSourceQueries
+      private
+
+      # @rbs (Frontend::SourceDocument document, Frontend::Location start,
+      #   Frontend::Location? finish) -> Array[Frontend::Token]
+      def tokens_between(document, start, finish)
+        document.tokens.select do |token|
+          after = !location_before?(token.location, start)
+          before = finish.nil? || location_before?(token.location, finish)
+          after && before
+        end
+      end
+
+      # @rbs (Frontend::SourceDocument document, Frontend::Location location, String value) -> Frontend::Token?
+      def token_at(document, location, value)
+        document.tokens.find do |token|
+          token.location.line == location.line && token.location.column == location.column && token.value == value
+        end
+      end
+
+      # @rbs (Frontend::SourceDocument document, Frontend::Token lhs_token) -> Array[Frontend::Token]
+      def parameter_tokens(document, lhs_token)
+        start_index = document.tokens.index(lhs_token)
+        return [] unless start_index
+
+        tail = document.tokens.drop(start_index + 1)
+        return [] unless tail.first&.type == :"("
+
+        tail.take_while { |token| token.type != :")" }.select { |token| token.type == :identifier }
+      end
+
+      # @rbs (Frontend::AST::declaration declaration) -> Array[String]
+      def declaration_reference_names(declaration)
+        case declaration
+        when Frontend::AST::Start, Frontend::AST::OnErrorReduce
+          declaration.names
+        when Frontend::AST::Recovery
+          declaration.sync_tokens
+        when Frontend::AST::DisplayName, Frontend::AST::SemanticType
+          [declaration.name]
+        when Frontend::AST::Convert
+          declaration.pairs.map(&:name)
+        when Frontend::AST::Precedence
+          declaration.levels.flat_map(&:symbols)
+        else
+          []
+        end
+      end
+
+      # @rbs (Frontend::Location left, Frontend::Location right) -> bool
+      def location_before?(left, right)
+        left.line < right.line || (left.line == right.line && left.column < right.column)
+      end
+    end
+  end
+end

data/lib/ibex/lsp/symbol_occurrence.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # A source-backed grammar symbol definition or reference.
+    SymbolOccurrence = Struct.new(
+      :name, #: String
+      :kind, #: Symbol
+      :role, #: Symbol
+      :key, #: Array[untyped]
+      :path, #: String
+      :span, #: Frontend::SourceSpan
+      :data, #: Hash[Symbol, untyped]
+      keyword_init: true
+    )
+  end
+end

data/lib/ibex/lsp/transport.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Reads and writes bounded LSP JSON-RPC messages over Content-Length framing.
+    class Transport
+      HEADER_SEPARATOR = "\r\n\r\n"
+
+      # @rbs (untyped input, untyped output) -> void
+      def initialize(input, output)
+        @input = input
+        @output = output
+        @buffer = String.new(encoding: Encoding::BINARY)
+      end
+
+      # @rbs () -> Hash[String, untyped]?
+      def read_message
+        header = read_header
+        return unless header
+
+        length = content_length(header)
+        body = read_bytes(length)
+        parsed = JSON.parse(body)
+        raise ProtocolError.new("JSON-RPC message must be an object", code: -32_600) unless parsed.is_a?(Hash)
+
+        parsed
+      rescue JSON::ParserError => e
+        raise ProtocolError.new("malformed JSON: #{e.message}", code: -32_700)
+      end
+
+      # @rbs (Hash[String, untyped] message) -> void
+      def write_message(message)
+        body = JSON.generate(message)
+        if body.bytesize > Limits::MAX_OUTPUT_BYTES
+          raise ProtocolError.new("JSON-RPC output exceeds #{Limits::MAX_OUTPUT_BYTES} bytes", code: -32_603)
+        end
+
+        @output.write("Content-Length: #{body.bytesize}\r\n\r\n")
+        @output.write(body)
+        @output.flush if @output.respond_to?(:flush)
+      end
+
+      private
+
+      # @rbs () -> String?
+      def read_header
+        loop do
+          index = @buffer.index(HEADER_SEPARATOR)
+          if index
+            if index > Limits::MAX_HEADER_BYTES
+              raise ProtocolError.new("JSON-RPC header exceeds #{Limits::MAX_HEADER_BYTES} bytes",
+                                      code: -32_700, fatal: true)
+            end
+
+            header = @buffer.byteslice(0, index) || ""
+            @buffer = @buffer.byteslice(index + HEADER_SEPARATOR.bytesize, @buffer.bytesize) || +""
+            return header
+          end
+          if @buffer.bytesize > Limits::MAX_HEADER_BYTES
+            raise ProtocolError.new("JSON-RPC header exceeds #{Limits::MAX_HEADER_BYTES} bytes",
+                                    code: -32_700, fatal: true)
+          end
+
+          chunk = read_chunk(4096)
+          return nil if chunk.nil? && @buffer.empty?
+          raise ProtocolError.new("truncated JSON-RPC header", code: -32_700, fatal: true) unless chunk
+
+          @buffer << chunk.b
+        end
+      end
+
+      # @rbs (String header) -> Integer
+      def content_length(header)
+        values = header.split("\r\n").filter_map do |line|
+          name, value = line.split(":", 2)
+          raise ProtocolError.new("malformed JSON-RPC header", code: -32_700, fatal: true) unless value
+
+          value.strip if name&.casecmp?("Content-Length")
+        end
+        unless values.one? && values.fetch(0).match?(/\A\d+\z/)
+          raise ProtocolError.new("exactly one numeric Content-Length header is required",
+                                  code: -32_700, fatal: true)
+        end
+
+        length = Integer(values.fetch(0), 10)
+        if length > Limits::MAX_MESSAGE_BYTES
+          raise ProtocolError.new("JSON-RPC message exceeds #{Limits::MAX_MESSAGE_BYTES} bytes",
+                                  code: -32_700, fatal: true)
+        end
+        length
+      end
+
+      # @rbs (Integer length) -> String
+      def read_bytes(length)
+        while @buffer.bytesize < length
+          chunk = read_chunk([4096, length - @buffer.bytesize].min)
+          raise ProtocolError.new("truncated JSON-RPC body", code: -32_700, fatal: true) unless chunk
+
+          @buffer << chunk.b
+        end
+        body = @buffer.byteslice(0, length) || ""
+        @buffer = @buffer.byteslice(length, @buffer.bytesize) || +""
+        body.force_encoding(Encoding::UTF_8)
+      end
+
+      # IO#read(length) may wait for the entire length on a live pipe. readpartial
+      # lets the stdio server process the bytes that are already available.
+      # @rbs (Integer length) -> String?
+      def read_chunk(length)
+        return @input.read(length) unless @input.respond_to?(:readpartial)
+
+        @input.readpartial(length)
+      rescue EOFError
+        nil
+      end
+    end
+  end
+end

data/lib/ibex/lsp/workspace.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Converts file URIs to canonical paths constrained to initialized workspace roots.
+    class Workspace
+      attr_reader :roots #: Array[String]
+
+      # @rbs (Array[String] root_uris, Frontend::SourceLoader loader) -> void
+      def initialize(root_uris, loader)
+        raise ProtocolError.new("initialize requires a workspace root", code: -32_602) if root_uris.empty?
+
+        @loader = loader
+        @roots = root_uris.map { |uri| canonical_root(uri) }.uniq.sort.freeze
+      end
+
+      # @rbs (String uri, ?allow_missing: bool) -> String
+      def path(uri, allow_missing: true)
+        parsed = parse_file_uri(uri)
+        decoded = decode_path(parsed.path)
+
+        canonical = @loader.canonical_path(decoded, allow_missing: allow_missing)
+        return canonical if root_for(canonical)
+
+        raise ProtocolError.new("document is outside initialized workspace roots: #{uri}", code: -32_602)
+      rescue URI::InvalidURIError, ArgumentError => e
+        raise ProtocolError.new("invalid file URI #{uri.inspect}: #{e.message}", code: -32_602)
+      rescue SystemCallError => e
+        raise ProtocolError.new("cannot resolve file URI #{uri.inspect}: #{e.message}", code: -32_602)
+      end
+
+      # @rbs (String path) -> String
+      def uri(path)
+        canonical = @loader.canonical_path(path, allow_missing: true)
+        parser = begin
+          URI::RFC2396_PARSER
+        rescue NameError
+          URI::DEFAULT_PARSER
+        end
+        "file://#{parser.escape(canonical)}"
+      end
+
+      # @rbs (String path) -> String?
+      def root_for(path)
+        roots.select { |root| inside?(path, root) }.max_by(&:length)
+      end
+
+      private
+
+      # URI is a stdlib boundary whose implementation class varies by scheme.
+      # @rbs (String uri) -> untyped
+      def parse_file_uri(uri)
+        validate_raw_authority!(uri)
+        parsed = URI.parse(uri)
+        path = parsed.path
+        local_host = parsed.host.nil? || parsed.host.empty? || parsed.host.casecmp?("localhost")
+        unless parsed.scheme == "file" && local_host &&
+               parsed.query.nil? && parsed.fragment.nil? && path&.start_with?("/")
+          raise ArgumentError, "only absolute local file URIs are supported"
+        end
+
+        parsed
+      end
+
+      # Ruby's URI::File may discard userinfo and port, so validate the raw authority first.
+      # @rbs (String uri) -> void
+      def validate_raw_authority!(uri)
+        match = uri.match(%r{\Afile://([^/]*)}i)
+        return unless match
+
+        authority = match[1] || ""
+        return if authority.empty? || authority.casecmp?("localhost")
+
+        raise ArgumentError, "only an empty authority or localhost is supported"
+      end
+
+      # @rbs (String uri) -> String
+      def canonical_root(uri)
+        parsed = parse_file_uri(uri)
+        decoded = decode_path(parsed.path)
+        canonical = File.realpath(decoded)
+        unless File.directory?(canonical)
+          raise ProtocolError.new("workspace root is not a directory: #{uri}", code: -32_602)
+        end
+
+        canonical
+      rescue URI::InvalidURIError, ArgumentError, SystemCallError => e
+        raise ProtocolError.new("invalid workspace root #{uri.inspect}: #{e.message}", code: -32_602)
+      end
+
+      # @rbs (String encoded) -> String
+      def decode_path(encoded)
+        raise ArgumentError, "file URI contains an invalid percent escape" if
+          encoded.match?(/%(?![0-9A-Fa-f]{2})/)
+        if encoded.match?(/%(?:2f|5c|00)/i)
+          raise ArgumentError, "file URI must not percent-encode a path separator or NUL"
+        end
+
+        parser = begin
+          URI::RFC2396_PARSER
+        rescue NameError
+          URI::DEFAULT_PARSER
+        end
+        decoded = parser.unescape(encoded)
+        raise ArgumentError, "file URI path must be valid UTF-8" unless decoded.valid_encoding?
+        raise ArgumentError, "file URI path must not contain NUL" if decoded.include?("\0")
+        raise ArgumentError, "file URI path must not contain parent traversal" if
+          decoded.split(%r{[\\/]}).include?("..")
+
+        decoded
+      end
+
+      # @rbs (String path, String root) -> bool
+      def inside?(path, root)
+        cursor = path
+        loop do
+          return true if cursor == root
+
+          parent = File.dirname(cursor)
+          return false if parent == cursor
+
+          cursor = parent
+        end
+      end
+    end
+  end
+end