ibex 0.1.0 → 0.3.0

data/lib/ibex/frontend/dsl.rb

@@ -48,9 +48,29 @@ module Ibex
           @declarations << AST::Expect.new(conflicts: conflicts, loc: next_location)
         end
 
-        # @rbs (Object name) -> void
-        def start(name)
-          @declarations << AST::Start.new(name: name.to_s, loc: next_location)
+        # @rbs (*Object names) -> void
+        def start(*names)
+          @declarations << AST::Start.new(names: names.map(&:to_s), loc: next_location)
+        end
+
+        # @rbs (*Object names) -> void
+        def recover_sync(*names)
+          @declarations << AST::Recovery.new(sync_tokens: names.map(&:to_s), loc: next_location)
+        end
+
+        # @rbs (*Object names) -> void
+        def on_error_reduce(*names)
+          @declarations << AST::OnErrorReduce.new(names: names.map(&:to_s), loc: next_location)
+        end
+
+        # @rbs (Object source) -> void
+        def test_accept(source)
+          @declarations << AST::GrammarTest.new(expectation: :accept, source: source.to_s, loc: next_location)
+        end
+
+        # @rbs (Object source) -> void
+        def test_reject(source)
+          @declarations << AST::GrammarTest.new(expectation: :reject, source: source.to_s, loc: next_location)
         end
 
         # @rbs (Object name, Object expression) -> void
@@ -60,6 +80,29 @@ module Ibex
           @declarations << AST::Convert.new(pairs: [pair], loc: location)
         end
 
+        # @rbs (Object name, Object value) -> void
+        def display(name, value)
+          @declarations << AST::DisplayName.new(name: name.to_s, value: metadata_value(value, "display"),
+                                                loc: next_location)
+        end
+
+        # @rbs (Object name, Object value) -> void
+        def type(name, value)
+          @declarations << AST::SemanticType.new(name: name.to_s, value: metadata_value(value, "type"),
+                                                 loc: next_location)
+        end
+
+        # @rbs (Object name, ?Object? semantic_type) -> void
+        def param(name, semantic_type = nil)
+          type = semantic_type && metadata_value(semantic_type, "%param")
+          @declarations << AST::Parameter.new(name: name.to_s, semantic_type: type, loc: next_location)
+        end
+
+        # @rbs (Object name, Object code) -> void
+        def printer(name, code)
+          @declarations << AST::Printer.new(name: name.to_s, code: code.to_s, loc: next_location)
+        end
+
         # @rbs (?direction: Symbol) { (PrecedenceBuilder) -> void } -> void
         def precedence(direction: :low_to_high)
           location = next_location
@@ -73,7 +116,9 @@ module Ibex
           location = next_location
           builder = RuleBuilder.new(self, location)
           yield builder
-          @rules << AST::Rule.new(lhs: lhs.to_s, alternatives: builder.alternatives, loc: location)
+          @rules << AST::Rule.new(
+            lhs: lhs.to_s, parameters: [], alternatives: builder.alternatives, loc: location, inline: false
+          )
         end
 
         # @rbs (Object name, Object code) -> void
@@ -134,6 +179,19 @@ module Ibex
           @line += 1
           location
         end
+
+        private
+
+        # @rbs (Object value, String feature) -> String
+        def metadata_value(value, feature)
+          string = value.to_s
+          raise ArgumentError, "#{feature} value must not be empty" if string.strip.empty?
+          raise ArgumentError, "#{feature} value must be a single line" if string.match?(/[\r\n]/)
+          raise ArgumentError, "#{feature} value must not contain control characters" if
+            string.match?(/[[:cntrl:]]/)
+
+          string
+        end
       end
 
       # Collects ordered associativity levels.
@@ -180,13 +238,14 @@ module Ibex
           @alternatives = [] #: Array[AST::Alternative]
         end
 
-        # @rbs (*Object items, ?action: Object?, ?precedence: Object?) -> void
-        def alt(*items, action: nil, precedence: nil)
+        # @rbs (*Object items, ?action: Object?, ?precedence: Object?, ?node: AST::NodeAnnotation?) -> void
+        def alt(*items, action: nil, precedence: nil, node: nil)
           location = @grammar.next_location || @default_location
           normalized = items.map { |item| @grammar.normalize_item(item) }
           action_node = action && AST::InlineAction.new(code: action.to_s, loc: location)
           alternative = AST::Alternative.new(
-            items: normalized, action: action_node, precedence: precedence&.to_s, loc: location
+            items: normalized, action: action_node, precedence: precedence&.to_s,
+            node_annotation: node, loc: location
           )
           @alternatives << alternative
         end

data/lib/ibex/frontend/formatter.rb

@@ -0,0 +1,453 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Frontend
+    # Deterministically formats grammar trivia and rejects any semantic change.
+    # rubocop:disable Metrics/ClassLength -- token layout policy stays together so adjacency rules remain auditable.
+    class Formatter
+      DECLARATION_STARTS = %i[
+        PRAGMA INCLUDE IMPORT TOKEN PRECHIGH PRECLOW OPTIONS EXPECT EXPECT_RR START RECOVER ON_ERROR_REDUCE TEST LEXER
+        CONVERT DISPLAY TYPE PARAM PRINTER PARSER RULE
+      ].freeze #: Array[Symbol]
+      ASSOCIATIONS = %i[LEFT RIGHT NONASSOC].freeze #: Array[Symbol]
+      CALLABLES = %i[LHS PARAMETERIZED_REFERENCE SEPARATED_LIST SEPARATED_NONEMPTY_LIST].freeze #: Array[Symbol]
+      SUFFIXES = %w[? * +].freeze #: Array[String]
+      CLOSERS = %w[) , ;].freeze #: Array[String]
+      ADJACENT_AFTER = ["::", "("].freeze #: Array[String]
+
+      # @rbs!
+      #   type formatter_element = {
+      #     segment: Segment,
+      #     external: external_token?,
+      #     role: Symbol?,
+      #     gap: Array[Segment]
+      #   }
+      #   type formatter_state = {
+      #     section: :declarations | :rules,
+      #     precedence_closer: external_token?,
+      #     conversion: bool,
+      #     conversion_name: bool,
+      #     parser: bool,
+      #     parser_key: bool,
+      #     depth: Integer,
+      #     rule_colon: bool
+      #   }
+
+      # @rbs (String source, ?file: String, ?mode: Symbol) -> String
+      def self.format(source, file: "(grammar)", mode: :default)
+        new(mode: mode).format(source, file: file)
+      end
+
+      # @rbs (?mode: Symbol) -> void
+      def initialize(mode: :default)
+        raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(mode)
+
+        @mode = mode
+      end
+
+      # @rbs (String source, ?file: String) -> String
+      def format(source, file: "(grammar)")
+        document = Parser.new(source, file: file, mode: @mode).parse_source_document
+        formatted = render(document)
+        reparsed = Parser.new(formatted, file: file, mode: @mode).parse_source_document
+        unless same_semantic_projection?(document.ast, reparsed.ast)
+          raise Ibex::Error, "#{file}: formatting would change grammar semantics"
+        end
+
+        formatted
+      rescue Ibex::Error
+        raise
+      rescue StandardError => e
+        raise Ibex::Error, "#{file}: formatting failed: #{e.message}"
+      end
+
+      private
+
+      # @rbs (SourceDocument document) -> String
+      def render(document)
+        @default_newline = document.source.match(/\r\n|\n/)&.[](0) || "\n"
+        elements = build_elements(document)
+        output = +""
+        previous = nil #: formatter_element?
+        elements.each do |current|
+          output << separator(previous, current)
+          output << current.fetch(:segment).text
+          previous = current
+        end
+        output
+      end
+
+      # @rbs (SourceDocument document) -> Array[formatter_element]
+      def build_elements(document)
+        external_types = external_types(document.tokens)
+        pending = [] #: Array[Segment]
+        elements = document.cst.each_with_object([]) do |segment, result|
+          if %i[whitespace newline].include?(segment.kind)
+            pending << segment
+            next
+          end
+
+          result << {
+            segment: segment,
+            external: segment.token_index && external_types[segment.token_index],
+            role: nil,
+            gap: pending
+          }
+          pending = []
+        end
+        annotate_roles(elements)
+        elements
+      end
+
+      # @rbs (Array[Token] tokens) -> Hash[Integer, external_token]
+      def external_types(tokens)
+        adapter = TokenAdapter.new(tokens, extended: @mode == :extended)
+        types = {} #: Hash[Integer, external_token]
+        index = 0
+        while (classified = adapter.next_token)
+          types[index] = classified.fetch(0)
+          index += 1
+        end
+        types
+      end
+
+      # @rbs (Array[formatter_element] elements) -> void
+      def annotate_roles(elements)
+        state = {
+          section: :declarations, precedence_closer: nil, conversion: false,
+          conversion_name: true, parser: false, parser_key: true, depth: 0, rule_colon: false
+        } #: formatter_state
+        elements.each do |element|
+          external = element.fetch(:external)
+          next unless external
+
+          element[:role] = token_role(external, state)
+          advance_role_state(external, element.fetch(:role), state)
+        end
+        annotate_comment_indentation(elements)
+      end
+
+      # @rbs (Array[formatter_element] elements) -> void
+      def annotate_comment_indentation(elements)
+        following_role = nil #: Symbol?
+        elements.reverse_each do |element|
+          segment = element.fetch(:segment)
+          if comment?(segment)
+            element[:role] = :rule_comment if following_role == :rule_start
+            element[:role] = :alternative_comment if following_role == :alternative
+          elsif element.fetch(:external)
+            following_role = element.fetch(:role)
+          end
+        end
+      end
+
+      # @rbs (external_token external, formatter_state state) -> Symbol?
+      def token_role(external, state)
+        document_role = document_token_role(external, state)
+        return document_role if document_role
+
+        rule_token_role(external, state)
+      end
+
+      # @rbs (external_token external, formatter_state state) -> Symbol?
+      def document_token_role(external, state)
+        return :user_code if external == :USER_CODE
+
+        nested_role = nested_declaration_role(external, state)
+        return nested_role if nested_role
+        return :conversion_entry if state.fetch(:conversion) && state.fetch(:conversion_name)
+        return :rule_keyword if external == :RULE
+        return :declaration if DECLARATION_STARTS.include?(external)
+
+        nil
+      end
+
+      # @rbs (external_token external, formatter_state state) -> Symbol?
+      def nested_declaration_role(external, state)
+        return precedence_role(external, state) if %i[PRECHIGH PRECLOW].include?(external)
+        return :precedence_level if ASSOCIATIONS.include?(external) && state[:precedence_closer]
+        return :parser_end if external == :END && state.fetch(:parser)
+        return :parser_setting if state.fetch(:parser) && state.fetch(:parser_key)
+        return section_end_role(state) if external == :END
+
+        nil
+      end
+
+      # @rbs (formatter_state state) -> Symbol?
+      def section_end_role(state)
+        return :conversion_end if state.fetch(:conversion)
+        return :grammar_end if state.fetch(:section) == :rules
+
+        nil
+      end
+
+      # @rbs (external_token external, formatter_state state) -> Symbol?
+      def rule_token_role(external, state)
+        return :rule_start if external == :INLINE && state.fetch(:section) == :rules
+        return :rule_start if external == :LHS
+        return :alternative if external == "|" && state.fetch(:section) == :rules && state.fetch(:depth).zero?
+
+        if external == ":"
+          return :rule_colon if state.fetch(:rule_colon) && state.fetch(:depth).zero?
+
+          return :named_reference
+        end
+
+        nil
+      end
+
+      # @rbs (external_token external, formatter_state state) -> Symbol
+      def precedence_role(external, state)
+        return :precedence_end if state[:precedence_closer] == external
+
+        :declaration
+      end
+
+      # @rbs (external_token external, Symbol? role, formatter_state state) -> void
+      def advance_role_state(external, role, state)
+        advance_precedence_state(external, role, state)
+        advance_conversion_state(external, role, state)
+        advance_parser_state(external, role, state)
+        advance_rule_state(external, role, state)
+      end
+
+      # @rbs (external_token external, Symbol? role, formatter_state state) -> void
+      def advance_parser_state(external, role, state)
+        if external == :PARSER
+          state[:parser] = true
+          state[:parser_key] = true
+        elsif role == :parser_end
+          state[:parser] = false
+        elsif state.fetch(:parser)
+          state[:parser_key] = !state.fetch(:parser_key)
+        end
+      end
+
+      # @rbs (external_token external, Symbol? role, formatter_state state) -> void
+      def advance_precedence_state(external, role, state)
+        if role == :declaration && %i[PRECHIGH PRECLOW].include?(external)
+          state[:precedence_closer] = external == :PRECHIGH ? :PRECLOW : :PRECHIGH
+        elsif role == :precedence_end
+          state[:precedence_closer] = nil
+        end
+      end
+
+      # @rbs (external_token external, Symbol? role, formatter_state state) -> void
+      def advance_conversion_state(external, role, state)
+        if external == :CONVERT
+          state[:conversion] = true
+          state[:conversion_name] = true
+        elsif role == :conversion_end
+          state[:conversion] = false
+        elsif state.fetch(:conversion) && role == :conversion_entry
+          state[:conversion_name] = false
+        elsif state.fetch(:conversion) && !state.fetch(:conversion_name)
+          state[:conversion_name] = true
+        end
+      end
+
+      # @rbs (external_token external, Symbol? role, formatter_state state) -> void
+      def advance_rule_state(external, role, state)
+        state[:section] = :rules if external == :RULE
+        state[:rule_colon] = true if external == :LHS
+        state[:rule_colon] = false if role == :rule_colon
+        state[:depth] += 1 if external == "("
+        state[:depth] -= 1 if external == ")"
+      end
+
+      # @rbs (formatter_element? previous, formatter_element current) -> String
+      def separator(previous, current)
+        gap = current.fetch(:gap)
+        return leading_separator(current, gap) unless previous
+
+        previous_segment = previous.fetch(:segment)
+        current_segment = current.fetch(:segment)
+        return "" if direct_opaque_pair?(previous_segment, current_segment)
+        return trailing_separator(previous, gap) if current_segment.kind == :eof
+        return comment_separator(previous, current, gap) if comment?(previous_segment) || comment?(current_segment)
+
+        kind = separator_kind(previous, current)
+        render_separator(kind, gap, indentation(current))
+      end
+
+      # @rbs (formatter_element current, Array[Segment] gap) -> String
+      def leading_separator(current, gap)
+        return "" unless newline_in?(gap)
+
+        render_separator(:line, gap, indentation(current))
+      end
+
+      # @rbs (formatter_element previous, Array[Segment] gap) -> String
+      def trailing_separator(previous, gap)
+        segment = previous.fetch(:segment)
+        return "" if segment.kind == :user_code_body
+        return render_separator(:line, gap, 0) if previous.fetch(:role) == :grammar_end
+
+        render_separator(gap.any? { |item| item.kind == :newline } ? :line : :none, gap, 0)
+      end
+
+      # @rbs (Segment previous, Segment current) -> bool
+      def direct_opaque_pair?(previous, current)
+        (previous.kind == :user_code_marker && current.kind == :user_code_body) ||
+          (previous.kind == :user_code_body && current.kind == :user_code_marker)
+      end
+
+      # @rbs (Segment segment) -> bool
+      def comment?(segment)
+        %i[line_comment block_comment].include?(segment.kind)
+      end
+
+      # @rbs (formatter_element previous, formatter_element current, Array[Segment] gap) -> String
+      def comment_separator(previous, current, gap)
+        previous_segment = previous.fetch(:segment)
+        current_segment = current.fetch(:segment)
+        if text_ends_with_newline?(previous_segment.text)
+          newlines = gap.select { |segment| segment.kind == :newline }.map(&:text)
+          return newlines.join + (" " * indentation(current))
+        end
+        if previous_segment.kind == :line_comment || newline_in?(gap)
+          return render_separator(:line, gap, indentation(current))
+        end
+        return " " if current_segment.kind == :line_comment || previous_segment.kind == :block_comment ||
+                      current_segment.kind == :block_comment
+
+        render_separator(:space, gap, indentation(current))
+      end
+
+      # @rbs (formatter_element previous, formatter_element current) -> Symbol
+      def separator_kind(previous, current)
+        previous_external = previous.fetch(:external)
+        current_external = current.fetch(:external)
+        previous_role = previous.fetch(:role)
+        current_role = current.fetch(:role)
+
+        return :line if line_boundary?(previous_role, current_role, previous, current)
+        return :none if adjacent_tokens?(previous_external, current_external, previous_role, current_role)
+
+        :space
+      end
+
+      # @rbs (external_token? previous_external, external_token? current_external,
+      #   Symbol? previous_role, Symbol? current_role) -> bool
+      def adjacent_tokens?(previous_external, current_external, previous_role, current_role)
+        return true if current_role == :named_reference || previous_role == :named_reference
+        return true if current_external == "::"
+        return true if CLOSERS.include?(current_external) || SUFFIXES.include?(current_external)
+        return true if ADJACENT_AFTER.include?(previous_external)
+
+        current_external == "(" && CALLABLES.include?(previous_external)
+      end
+
+      # @rbs (Symbol? previous_role, Symbol? current_role, formatter_element previous,
+      #   formatter_element current) -> bool
+      def line_boundary?(previous_role, current_role, previous, current)
+        return false if current_role == :rule_start && previous.fetch(:external) == :INLINE
+
+        return true if %i[declaration rule_keyword precedence_level precedence_end conversion_entry
+                          conversion_end parser_setting parser_end rule_start alternative grammar_end
+                          user_code].include?(current_role)
+        return true if previous_role == :rule_keyword
+        return false unless current.fetch(:segment).kind == :user_code_marker
+
+        previous.fetch(:segment).kind != :user_code_body
+      end
+
+      # @rbs (formatter_element element) -> Integer
+      def indentation(element)
+        case element.fetch(:role)
+        when :rule_start, :precedence_level, :conversion_entry, :parser_setting, :rule_comment then 2
+        when :alternative, :alternative_comment then 4
+        else 0
+        end
+      end
+
+      # @rbs (Symbol kind, Array[Segment] gap, Integer indentation) -> String
+      def render_separator(kind, gap, indentation)
+        return "" if kind == :none
+        return " " if kind == :space
+
+        newlines = gap.select { |segment| segment.kind == :newline }.map(&:text)
+        newlines = [@default_newline] if newlines.empty?
+        newlines.join + (" " * indentation)
+      end
+
+      # @rbs (Array[Segment] gap) -> bool
+      def newline_in?(gap)
+        gap.any? { |segment| segment.kind == :newline }
+      end
+
+      # @rbs (String text) -> bool
+      def text_ends_with_newline?(text)
+        text.end_with?("\n")
+      end
+
+      # @rbs (Object? left, Object? right) -> bool
+      def same_semantic_projection?(left, right)
+        pending = [[left, right]] #: Array[[Object?, Object?]]
+        until pending.empty?
+          pair = pending.pop
+          next unless pair
+
+          left_value, right_value = pair
+          next if left_value.equal?(right_value)
+          return false unless comparable_semantic_values?(left_value, right_value)
+
+          append_semantic_children(pending, left_value, right_value)
+        end
+        true
+      end
+
+      # @rbs (Object? left, Object? right) -> bool
+      def comparable_semantic_values?(left, right)
+        if left.is_a?(Struct)
+          return false unless right.is_a?(Struct) && left.instance_of?(right.class)
+
+          return left.members == right.members
+        end
+        return right.is_a?(Array) && left.length == right.length if left.is_a?(Array)
+        return comparable_semantic_hashes?(left, right) if left.is_a?(Hash)
+
+        left == right
+      end
+
+      # @rbs (Hash[Object?, Object?] left, Object? right) -> bool
+      def comparable_semantic_hashes?(left, right)
+        return false unless right.is_a?(Hash)
+
+        left_keys = semantic_keys(left)
+        right_keys = semantic_keys(right)
+        left_keys.length == right_keys.length && left_keys.all? { |key| right_keys.include?(key) }
+      end
+
+      # @rbs (Array[[Object?, Object?]] pending, Object? left, Object? right) -> void
+      def append_semantic_children(pending, left, right)
+        case left
+        when Struct
+          return unless right.is_a?(Struct)
+
+          semantic_members(left).reverse_each { |member| pending << [left[member], right[member]] }
+        when Array
+          return unless right.is_a?(Array)
+
+          left.each_index.reverse_each { |index| pending << [left[index], right[index]] }
+        when Hash
+          return unless right.is_a?(Hash)
+
+          semantic_keys(left).reverse_each { |key| pending << [left[key], right[key]] }
+        end
+      end
+
+      # @rbs (Struct[Object?] value) -> Array[Symbol]
+      def semantic_members(value)
+        value.members.reject { |member| %i[loc span extended_loc].include?(member) }
+      end
+
+      # @rbs (Hash[Object?, Object?] value) -> Array[Object?]
+      def semantic_keys(value)
+        value.keys.reject { |key| %i[loc span].include?(key) }
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end