ibex 0.1.0 → 0.3.0

data/lib/ibex/lsp/workspace_analyzer.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Parses workspace sources, discovers safe include closures, and renders diagnostics.
+    class WorkspaceAnalyzer
+      GLOB_CHARACTERS = /[*?\[\]{}]/ #: Regexp
+      WINDOWS_ABSOLUTE = %r{\A(?:[A-Za-z]:[\\/]|\\\\)} #: Regexp
+
+      # @rbs!
+      #   type lsp_diagnostic = lsp_object
+      #   type analyzed_document = {
+      #     source: String,
+      #     document: Frontend::SourceDocument?,
+      #     diagnostics: Array[lsp_diagnostic]
+      #   }
+
+      # @rbs (Workspace workspace, Frontend::SourceLoader loader) -> void
+      def initialize(workspace, loader)
+        @workspace = workspace
+        @loader = loader
+      end
+
+      # @rbs (String path) -> analyzed_document
+      def analyze(path)
+        source = @loader.read(path)
+        return analyze_fragment(path, source) if fragment_source?(source, path)
+
+        analyze_root(path, source)
+      rescue Ibex::Error, SystemCallError => e
+        source ||= readable_source(path)
+        { source: source, document: nil, diagnostics: [diagnostic(e.message, source, path)] }
+      end
+
+      # @rbs (String root) -> [Array[String], Hash[String, analyzed_document]]
+      def closure(root)
+        files = [] #: Array[String]
+        analyzed = {} #: Hash[String, analyzed_document]
+        visiting = [root]
+        until visiting.empty?
+          path = visiting.pop
+          next unless path
+          next if analyzed.key?(path)
+
+          current = analyze(path)
+          analyzed[path] = current
+          files << path
+          dependencies(current.fetch(:document)).reverse_each do |target|
+            visiting << target unless analyzed.key?(target)
+          end
+        end
+        [files, analyzed]
+      end
+
+      # @rbs (String root) -> [Frontend::Resolution?, Array[lsp_diagnostic]]
+      def resolve(root)
+        resolution = Frontend::Resolver.new(root, mode: :extended, loader: @loader).resolve
+        [resolution, []]
+      rescue Ibex::Error, SystemCallError => e
+        [nil, [diagnostic(e.message, source_for_error(e.message, root), file_for_error(e.message, root))]]
+      end
+
+      private
+
+      # @rbs (String path, String source) -> analyzed_document
+      def analyze_root(path, source)
+        result = Frontend::Parser.new(source, file: path, mode: :extended)
+                                 .parse_with_diagnostics(max_diagnostics: 20)
+        diagnostics = result.diagnostics.map { |entry| frontend_diagnostic(entry, source) }
+        { source: source, document: result.document, diagnostics: diagnostics }
+      end
+
+      # @rbs (String path, String source) -> analyzed_document
+      def analyze_fragment(path, source)
+        document = Frontend::Parser.new(source, file: path, mode: :extended).parse_source_document
+        { source: source, document: document, diagnostics: [] }
+      end
+
+      # @rbs (String source, String path) -> bool
+      def fragment_source?(source, path)
+        document, = Frontend::Lexer.new(source, file: path).tokenize_document_recovering(max_diagnostics: 1)
+        first = document.tokens.find { |token| token.type != :eof }
+        first&.value == "fragment"
+      rescue Ibex::Error
+        false
+      end
+
+      # @rbs (Frontend::SourceDocument? document) -> Array[String]
+      def dependencies(document)
+        ast = document&.ast
+        return [] unless ast
+
+        ast.declarations.filter_map do |declaration|
+          next unless declaration.is_a?(Frontend::AST::Include)
+
+          canonical_include(declaration)
+        rescue Ibex::Error, SystemCallError
+          nil
+        end
+      end
+
+      # @rbs (Frontend::AST::Include include_node) -> String
+      def canonical_include(include_node)
+        path = include_node.path
+        return invalid_include(include_node) if path.empty? || path.include?("\0")
+        return invalid_include(include_node) if path.start_with?("/") || path.match?(WINDOWS_ABSOLUTE)
+        return invalid_include(include_node) if path.split(%r{[\\/]}).include?("..")
+        return invalid_include(include_node) if path.match?(GLOB_CHARACTERS)
+
+        candidate = File.expand_path(path, File.dirname(include_node.loc.file))
+        canonical = @loader.canonical_path(candidate, allow_missing: true)
+        return canonical if @workspace.root_for(canonical)
+
+        invalid_include(include_node)
+      end
+
+      # @rbs (Frontend::AST::Include include_node) -> bot
+      def invalid_include(include_node)
+        raise Ibex::Error, "#{include_node.loc}: unsafe include path"
+      end
+
+      # @rbs (String message, String source, String fallback_path) -> lsp_diagnostic
+      def diagnostic(message, source, fallback_path)
+        file, line, column, text = diagnostic_parts(message, fallback_path)
+        {
+          "range" => diagnostic_range(source, line, column),
+          "severity" => 1,
+          "code" => "ibex.frontend",
+          "source" => "ibex",
+          "message" => text,
+          "data" => { "file" => file }
+        }
+      end
+
+      # @rbs (Frontend::Diagnostic diagnostic, String source) -> lsp_diagnostic
+      def frontend_diagnostic(diagnostic, source)
+        range = if diagnostic.span
+                  PositionCodec.new(source).range(diagnostic.span)
+                else
+                  diagnostic_range(source, diagnostic.location.line, diagnostic.location.column)
+                end
+        {
+          "range" => range,
+          "severity" => 1,
+          "code" => diagnostic.code,
+          "source" => "ibex",
+          "message" => diagnostic.message,
+          "data" => { "file" => diagnostic.location.file }
+        }
+      rescue ArgumentError, Ibex::Error
+        {
+          "range" => diagnostic_range("", 1, 1),
+          "severity" => 1,
+          "code" => diagnostic.code,
+          "source" => "ibex",
+          "message" => diagnostic.message,
+          "data" => { "file" => diagnostic.location.file }
+        }
+      end
+
+      # @rbs (String message, String fallback_path) -> [String, Integer, Integer, String]
+      def diagnostic_parts(message, fallback_path)
+        match = message.match(/\A(.*):(\d+):(\d+):\s*(.*)\z/m)
+        return [fallback_path, 1, 1, message] unless match
+
+        [match[1] || fallback_path, Integer(match[2]), Integer(match[3]), match[4] || message]
+      end
+
+      # @rbs (String source, Integer line, Integer column) -> Hash[String, Hash[String, Integer]]
+      def diagnostic_range(source, line, column)
+        line_text = source.lines.fetch(line - 1, "").delete_suffix("\n").delete_suffix("\r")
+        prefix = line_text.each_char.take([column - 1, 0].max).join
+        character = prefix.each_codepoint.sum { |codepoint| codepoint > 0xFFFF ? 2 : 1 }
+        point = { "line" => [line - 1, 0].max, "character" => character }
+        { "start" => point, "end" => point.dup }
+      end
+
+      # @rbs (String path) -> String
+      def readable_source(path)
+        @loader.read(path)
+      rescue SystemCallError
+        ""
+      end
+
+      # @rbs (String message, String fallback) -> String
+      def file_for_error(message, fallback)
+        match = message.match(/\A(.*):\d+:\d+:/)
+        return fallback unless match
+
+        match[1] || fallback
+      end
+
+      # @rbs (String message, String fallback) -> String
+      def source_for_error(message, fallback)
+        readable_source(file_for_error(message, fallback))
+      end
+    end
+  end
+end

data/lib/ibex/lsp.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+require_relative "version"
+require_relative "error"
+require_relative "frontend"
+require_relative "lsp/protocol_error"
+require_relative "lsp/transport"
+require_relative "lsp/position_codec"
+require_relative "lsp/workspace"
+require_relative "lsp/workspace_analyzer"
+require_relative "lsp/document_store_validation"
+require_relative "lsp/document_store_diagnostics"
+require_relative "lsp/document_store"
+require_relative "lsp/symbol_occurrence"
+require_relative "lsp/symbol_index_source_queries"
+require_relative "lsp/symbol_index_precedence_references"
+require_relative "lsp/symbol_index_builder"
+require_relative "lsp/symbol_index"
+require_relative "lsp/parser_configuration_assistance"
+require_relative "lsp/request_support"
+require_relative "lsp/initialization_handlers"
+require_relative "lsp/document_handlers"
+require_relative "lsp/navigation_handlers"
+require_relative "lsp/request_handlers"
+require_relative "lsp/server"
+
+module Ibex
+  # Language Server Protocol support for grammar workspaces.
+  module LSP
+  end
+end

data/lib/ibex/messages/en.yml

@@ -0,0 +1,28 @@
+{
+  "version": 1,
+  "messages": {
+    "cli.error": "%{detail}",
+    "diagnostic.frontend.lexical_error": "%{detail}",
+    "diagnostic.frontend.syntax_error": "%{detail}",
+    "diagnostic.frontend.resolution_error": "%{detail}",
+    "diagnostic.generic": "%{detail}",
+    "label.warning": "warning",
+    "label.warning_as_error": "warning treated as error",
+    "warning.undeclared_terminal": "undeclared terminal %{symbol}",
+    "warning.unused_terminal": "unused terminal %{symbol}",
+    "warning.unused_precedence": "unused precedence %{symbol}",
+    "warning.unreachable_terminal": "declared terminal %{symbol} is unreachable",
+    "warning.unreachable_nonterminal": "unreachable nonterminal %{symbol}",
+    "warning.duplicate_production": "duplicate production %{production} (first defined as %{original})",
+    "warning.implicit_empty": "implicit empty alternative; write %%empty to document intent",
+    "warning.empty_language": "start symbol %{symbol} derives no terminal sentence",
+    "warning.lexer_redos": "lexer pattern may exhibit excessive backtracking",
+    "warning.lexer_redos_symbol": "lexer pattern for %{symbol} may exhibit excessive backtracking",
+    "conflict.shift_reduce_expected": "%{actual} shift/reduce conflicts; expected %{expected}",
+    "conflict.reduce_reduce_expected": "%{actual} reduce/reduce conflicts; expected %{expected}",
+    "conflict.reduce_reduce": "%{actual} reduce/reduce conflicts",
+    "conflict.treated_as_error": "%{detail}; conflict treated as error",
+    "conflict.count": "%{count} %{kind} conflict%{plural}",
+    "note.ielr": "note: --algorithm=ielr avoids %{avoided}; consider --algorithm=ielr"
+  }
+}

data/lib/ibex/messages/ja.yml

@@ -0,0 +1,28 @@
+{
+  "version": 1,
+  "messages": {
+    "cli.error": "エラー: %{detail}",
+    "diagnostic.frontend.lexical_error": "字句エラー: %{detail}",
+    "diagnostic.frontend.syntax_error": "構文エラー: %{detail}",
+    "diagnostic.frontend.resolution_error": "解決エラー: %{detail}",
+    "diagnostic.generic": "診断: %{detail}",
+    "label.warning": "警告",
+    "label.warning_as_error": "エラーとして扱われた警告",
+    "warning.undeclared_terminal": "未宣言の終端記号 %{symbol}",
+    "warning.unused_terminal": "未使用の終端記号 %{symbol}",
+    "warning.unused_precedence": "未使用の優先順位 %{symbol}",
+    "warning.unreachable_terminal": "宣言された終端記号 %{symbol} に到達できません",
+    "warning.unreachable_nonterminal": "非終端記号 %{symbol} に到達できません",
+    "warning.duplicate_production": "生成規則 %{production} は重複しています（最初の定義: %{original}）",
+    "warning.implicit_empty": "暗黙の空選択肢です。意図を示すには %%empty を記述してください",
+    "warning.empty_language": "開始記号 %{symbol} は終端記号列を導出しません",
+    "warning.lexer_redos": "字句パターンで過大なバックトラックが発生する可能性があります",
+    "warning.lexer_redos_symbol": "%{symbol} の字句パターンで過大なバックトラックが発生する可能性があります",
+    "conflict.shift_reduce_expected": "shift/reduce 競合は %{actual} 件です（期待値: %{expected}）",
+    "conflict.reduce_reduce_expected": "reduce/reduce 競合は %{actual} 件です（期待値: %{expected}）",
+    "conflict.reduce_reduce": "reduce/reduce 競合が %{actual} 件あります",
+    "conflict.treated_as_error": "%{detail}（競合をエラーとして扱いました）",
+    "conflict.count": "%{kind} 競合 %{count} 件%{plural}",
+    "note.ielr": "注記: --algorithm=ielr により %{avoided} を回避できます。--algorithm=ielr を検討してください"
+  }
+}

data/lib/ibex/messages.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "json"
+
+module Ibex
+  # Immutable built-in diagnostic translations, separate from user E00xx
+  # sentence-keyed parser error catalogs.
+  module Messages
+    ROOT = File.expand_path("messages", __dir__ || raise("message catalog directory is unavailable")) #: String
+    LANGUAGES = %w[en ja].freeze #: Array[String]
+
+    # @rbs (String? requested) -> String
+    def language(requested)
+      candidate = requested.to_s.downcase.tr("_", "-").split("-").first
+      LANGUAGES.include?(candidate) ? candidate : "en"
+    end
+    module_function :language
+
+    # @rbs (String id, ?language: String, **Object? values) -> String
+    def translate(id, language: "en", **values)
+      selected = catalog(self.language(language))
+      template = selected[id] || catalog("en").fetch(id) do
+        raise Ibex::Error, "(messages):1:1: unknown built-in message id #{id.inspect}"
+      end
+      template % values.transform_keys(&:to_sym)
+    rescue KeyError => e
+      raise Ibex::Error, "(messages):1:1: missing interpolation for #{id.inspect}: #{e.message}"
+    end
+    module_function :translate
+
+    # @rbs (String id) -> bool
+    def known?(id)
+      catalog("en").key?(id)
+    end
+    module_function :known?
+
+    # @rbs (String language) -> Hash[String, String]
+    def catalog(language)
+      @catalogs ||= {} #: Hash[String, Hash[String, String]]
+      @catalogs[language] ||= load_catalog(language)
+    end
+    module_function :catalog
+
+    # @rbs (String language) -> Hash[String, String]
+    def load_catalog(language)
+      path = File.join(ROOT, "#{language}.yml")
+      document = JSON.parse(File.binread(path))
+      unless document.is_a?(Hash) && document["version"] == 1 && document["messages"].is_a?(Hash)
+        raise Ibex::Error, "#{path}:1:1: invalid built-in message catalog"
+      end
+
+      document.fetch("messages").to_h do |id, value|
+        unless id.is_a?(String) && value.is_a?(String)
+          raise Ibex::Error, "#{path}:1:1: message ids and templates must be strings"
+        end
+
+        [id.freeze, value.freeze]
+      end.freeze
+    rescue Errno::ENOENT => e
+      raise Ibex::Error, "#{path}:1:1: cannot read built-in message catalog: #{e.message}"
+    rescue JSON::ParserError => e
+      raise Ibex::Error, "#{path}:1:1: invalid JSON-compatible YAML: #{e.message}"
+    end
+    module_function :load_catalog
+    private_class_method :load_catalog
+  end
+end

data/lib/ibex/metrics.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "set"
+
+module Ibex
+  # Deterministic structural metrics over normalized Grammar and Automaton IR.
+  class Metrics
+    # @rbs!
+    #   type report_value = String | Integer | Float | Array[report_value] | Hash[Symbol, report_value]
+
+    # @rbs (IR::Automaton automaton) -> void
+    def initialize(automaton)
+      @automaton = automaton
+      @grammar = automaton.grammar
+    end
+
+    # @rbs () -> Hash[Symbol, report_value]
+    def to_h
+      {
+        ibex_report: "metrics", schema_version: 1,
+        algorithm: @automaton.algorithm,
+        grammar_digest: @automaton.grammar_digest,
+        symbols: symbol_metrics,
+        grammar: grammar_metrics,
+        automaton: automaton_metrics
+      }
+    end
+
+    private
+
+    # @rbs () -> Hash[Symbol, Integer]
+    def symbol_metrics
+      { terminals: @grammar.terminals.length, nonterminals: @grammar.nonterminals.length }
+    end
+
+    # @rbs () -> Hash[Symbol, report_value]
+    def grammar_metrics
+      rule_counts = @grammar.productions.group_by(&:lhs).transform_values(&:length)
+      {
+        rules: rule_counts.length,
+        alternatives: @grammar.productions.length,
+        average_alternatives_per_rule: average(rule_counts.values),
+        maximum_branching: rule_counts.values.max || 0,
+        epsilon_productions: @grammar.productions.count { |production| production.rhs.empty? },
+        recursive_nonterminals: recursive_nonterminal_ids.map { |id| symbol_name(id) }.sort,
+        recursion_component_depth: recursion_component_depth,
+        warnings: @grammar.warnings.length
+      }
+    end
+
+    # @rbs () -> Hash[Symbol, report_value]
+    def automaton_metrics
+      {
+        states: @automaton.states.length,
+        transitions: @automaton.states.sum { |state| state.transitions.length },
+        action_cells: @automaton.states.sum { |state| state.actions.length },
+        goto_cells: @automaton.states.sum { |state| state.gotos.length },
+        default_reductions: @automaton.states.count(&:default_action),
+        conflicts: {
+          shift_reduce: @automaton.conflict_summary.fetch(:sr),
+          precedence_resolved_shift_reduce: @automaton.conflict_summary.fetch(:resolved_sr),
+          reduce_reduce: @automaton.conflict_summary.fetch(:rr)
+        }
+      }
+    end
+
+    # @rbs (Array[Integer] values) -> Float
+    def average(values)
+      return 0.0 if values.empty?
+
+      values.sum.fdiv(values.length).round(6)
+    end
+
+    # @rbs () -> Hash[Integer, Array[Integer]]
+    def dependency_graph
+      @dependency_graph ||= begin
+        graph = {} #: Hash[Integer, Array[Integer]]
+        @grammar.nonterminals.each { |symbol| graph[symbol.id] = [] }
+        @grammar.productions.each do |production|
+          production.rhs.each do |id|
+            symbol = @grammar.symbol_by_id(id)
+            graph.fetch(production.lhs) << id if symbol&.nonterminal?
+          end
+        end
+        graph.transform_values { |targets| targets.uniq.sort }
+      end
+    end
+
+    # @rbs () -> Hash[Integer, Set[Integer]]
+    def reachability
+      @reachability ||= dependency_graph.to_h do |source, _targets|
+        found = Set[source]
+        queue = [source]
+        cursor = 0
+        while cursor < queue.length
+          dependency_graph.fetch(queue.fetch(cursor)).each do |target|
+            next if found.include?(target)
+
+            found << target
+            queue << target
+          end
+          cursor += 1
+        end
+        [source, found]
+      end
+    end
+
+    # @rbs () -> Array[Array[Integer]]
+    def recursion_components
+      remaining = dependency_graph.keys.sort
+      components = [] #: Array[Array[Integer]]
+      until remaining.empty?
+        source = remaining.fetch(0)
+        component = remaining.select do |candidate|
+          reachability.fetch(source).include?(candidate) && reachability.fetch(candidate).include?(source)
+        end
+        components << component
+        remaining -= component
+      end
+      components
+    end
+
+    # @rbs () -> Array[Integer]
+    def recursive_nonterminal_ids
+      recursion_components.flat_map do |component|
+        next component if component.length > 1
+
+        id = component.fetch(0)
+        dependency_graph.fetch(id).include?(id) ? component : []
+      end
+    end
+
+    # Longest path after collapsing mutually recursive nonterminals.
+    # @rbs () -> Integer
+    def recursion_component_depth
+      components = recursion_components
+      component_for = {} #: Hash[Integer, Integer]
+      components.each_with_index { |ids, index| ids.each { |id| component_for[id] = index } }
+      edges = Array.new(components.length) { Set.new } #: Array[Set[Integer]]
+      dependency_graph.each do |source, targets|
+        targets.each do |target|
+          from = component_for.fetch(source)
+          to = component_for.fetch(target)
+          edges.fetch(from) << to unless from == to
+        end
+      end
+      longest_dag_path(edges)
+    end
+
+    # @rbs (Array[Set[Integer]] edges) -> Integer
+    def longest_dag_path(edges)
+      indegree = Array.new(edges.length, 0)
+      edges.each { |targets| targets.each { |target| indegree[target] += 1 } }
+      depths = Array.new(edges.length, 1)
+      queue = indegree.each_index.select { |index| indegree[index].zero? }
+      cursor = 0
+      while cursor < queue.length
+        source = queue.fetch(cursor)
+        edges.fetch(source).each do |target|
+          depths[target] = [depths[target], depths[source] + 1].max
+          indegree[target] -= 1
+          queue << target if indegree[target].zero?
+        end
+        cursor += 1
+      end
+      depths.max || 0
+    end
+
+    # @rbs (Integer id) -> String
+    def symbol_name(id)
+      @grammar.symbol_by_id(id)&.name || id.to_s
+    end
+  end
+end