ibex 0.1.0

data/lib/ibex/runtime/parser.rb

@@ -0,0 +1,360 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    # Parser-table shape understood by this runtime.
+    PARSER_TABLE_FORMAT_VERSION = 1 #: Integer
+
+    # Raised by the default parser error handler.
+    class ParseError < StandardError; end
+
+    # rubocop:disable Metrics/ClassLength
+
+    # Drives a table-defined LR parser without native extensions.
+    #
+    # Subclasses provide `.parser_tables`, returning `:tokens`, `:token_names`,
+    # `:actions`, `:gotos`, and `:productions`. Actions are represented by
+    # `[:shift, state]`, `[:reduce, production]`, `[:accept]`, or `[:error]`.
+    class Parser
+      EOF_TOKEN = 0 #: Integer
+      ERROR_TOKEN = 1 #: Integer
+      NO_LOOKAHEAD = Object.new.freeze #: Object
+      RECOVERY_SHIFTS = 3 #: Integer
+      empty_row = {} # @type var empty_row: Hash[Integer, untyped]
+
+      EMPTY_ROW = empty_row.freeze #: Hash[Integer, untyped]
+
+      # @rbs @yydebug: bool
+      # @rbs @yydebug_output: IO
+      # @rbs @source: (^() -> untyped)?
+      # @rbs @state_stack: Array[Integer]
+      # @rbs @value_stack: Array[untyped]
+      # @rbs @lookahead: untyped
+      # @rbs @lookahead_value: untyped
+      # @rbs @recovery_shifts: Integer
+      # @rbs @semantic_error: bool
+      # @rbs @accept_requested: bool
+      # @rbs @unknown_token_id: Integer?
+      # @rbs @unknown_token_name: String?
+
+      attr_accessor :yydebug #: bool
+      attr_writer :yydebug_output #: IO
+
+      # @rbs () -> void
+      def initialize
+        @yydebug = false
+        @yydebug_output = $stderr
+        @source = nil
+        @state_stack = []
+        @value_stack = []
+        @lookahead = NO_LOOKAHEAD
+        @lookahead_value = nil
+        @recovery_shifts = 0
+        @semantic_error = false
+        @accept_requested = false
+        @unknown_token_id = nil
+      end
+
+      # Pull tokens from `next_token` and parse them.
+      # @rbs () -> untyped
+      def do_parse
+        drive_parser(-> { next_token })
+      end
+
+      # Parse tokens yielded by `receiver.method_id`.
+      # @rbs (untyped receiver, Symbol method_id) -> untyped
+      def yyparse(receiver, method_id)
+        stream = Enumerator.new do |tokens|
+          receiver.__send__(method_id) { |token| tokens << token }
+        end
+        drive_parser(-> { stream.next })
+      end
+
+      # Override in pull parsers. Return `[token, value]`, `false`, or `nil`.
+      # @rbs () -> ([untyped, untyped] | false | nil)
+      def next_token
+        raise NotImplementedError, "(input):1:1: next_token must be implemented"
+      end
+
+      # Override to recover from syntax errors. The default always raises.
+      # @rbs (Integer token_id, untyped value, Array[untyped] value_stack) -> untyped
+      def on_error(token_id, value, _value_stack)
+        expected = expected_tokens
+        suffix = expected.empty? ? "" : "; expected #{expected.join(', ')}"
+        raise ParseError, "(input):1:1: unexpected #{token_to_str(token_id)}#{suffix} (#{value.inspect})"
+      end
+
+      # Called after an ordinary input token is shifted. Override to observe
+      # the internal token id, semantic value, and destination state.
+      # @rbs (Integer token_id, untyped value, Integer state) -> void
+      def on_shift(_token_id, _value, _state); end
+
+      # Called after a production's semantic action and goto are committed.
+      # Override to observe its id, RHS values, and semantic result.
+      # @rbs (Integer production_id, Array[untyped] values, untyped result) -> void
+      def on_reduce(_production_id, _values, _result); end
+
+      # Called after the synthetic error token enters a recovery state.
+      # The payload describes the original error before recovery popped stacks.
+      # @rbs (Integer token_id, untyped value, Array[untyped] value_stack) -> void
+      def on_error_recover(_token_id, _value, _value_stack); end
+
+      # Return a human-readable name for an internal token id.
+      # @rbs (Integer token_id) -> String
+      def token_to_str(token_id)
+        return @unknown_token_name || token_id.to_s if token_id == @unknown_token_id
+
+        parser_tables.fetch(:token_names).fetch(token_id, token_id.to_s)
+      end
+
+      # Enter error recovery from a semantic action without calling `on_error`.
+      # @rbs () -> nil
+      def yyerror
+        @semantic_error = true
+        nil
+      end
+
+      # Leave error recovery immediately.
+      # @rbs () -> nil
+      def yyerrok
+        @recovery_shifts = 0
+        nil
+      end
+
+      # Accept immediately after the current semantic action completes.
+      # @rbs () -> nil
+      def yyaccept
+        @accept_requested = true
+        nil
+      end
+
+      # Return token names accepted in the current parser state.
+      # @rbs () -> Array[String]
+      def expected_tokens
+        return [] if @state_stack.empty?
+
+        state = @state_stack.last
+        parser_tables.fetch(:token_names).keys.filter_map do |token_id|
+          action = table_lookup(parser_tables.fetch(:actions), state, token_id) || default_action(state) || [:error]
+          token_to_str(token_id) unless error_action?(action) || token_id == ERROR_TOKEN
+        end
+      end
+
+      private
+
+      # @rbs (^() -> untyped source) -> untyped
+      def drive_parser(source)
+        prepare_parse(source)
+        loop do
+          action = action_for_current_state
+          outcome = perform(action)
+          return outcome[1] if outcome[0] == :done
+        end
+      ensure
+        @source = nil
+      end
+
+      # @rbs (^() -> untyped source) -> void
+      def prepare_parse(source)
+        validate_parser_table_format!
+        @source = source
+        @state_stack = [0]
+        @value_stack = []
+        @lookahead = NO_LOOKAHEAD
+        @lookahead_value = nil
+        @recovery_shifts = 0
+        @semantic_error = false
+        @accept_requested = false
+        @unknown_token_id = nil
+        trace("start state 0")
+      end
+
+      # @rbs () -> void
+      def validate_parser_table_format!
+        tables = parser_tables
+        unless tables.key?(:format_version)
+          raise ParseError,
+                "(tables):1:1: parser tables for #{self.class} are missing :format_version; " \
+                "regenerate the parser with the installed Ibex version"
+        end
+
+        actual = tables.fetch(:format_version)
+        return if actual == PARSER_TABLE_FORMAT_VERSION
+
+        raise ParseError,
+              "(tables):1:1: unsupported parser table format version #{actual.inspect} for #{self.class}; " \
+              "runtime supports #{PARSER_TABLE_FORMAT_VERSION}; regenerate the parser with the installed Ibex version"
+      end
+
+      # @rbs () -> untyped
+      def action_for_current_state
+        read_lookahead if @lookahead.equal?(NO_LOOKAHEAD)
+        state = @state_stack.last
+        return [:error] unless parser_tables.fetch(:token_names).key?(@lookahead)
+
+        explicit = table_lookup(parser_tables.fetch(:actions), state, @lookahead)
+        return explicit if explicit
+
+        default_action(state) || [:error]
+      end
+
+      # @rbs (untyped action) -> untyped
+      def perform(action)
+        case action.first
+        when :shift then shift(action.fetch(1))
+        when :reduce then reduce(action.fetch(1))
+        when :accept then [:done, @value_stack.last]
+        when :error then recover
+        else raise ParseError, "(tables):1:1: unknown parser action #{action.inspect}"
+        end
+      end
+
+      # @rbs (Integer next_state) -> untyped
+      def shift(next_state)
+        token_id = @lookahead
+        value = @lookahead_value
+        trace("shift #{token_to_str(token_id)} -> state #{next_state}")
+        @state_stack << next_state
+        @value_stack << value
+        @lookahead = NO_LOOKAHEAD
+        @recovery_shifts -= 1 if @recovery_shifts.positive?
+        on_shift(token_id, value, next_state)
+        [:continue]
+      end
+
+      # @rbs (Integer production_id) -> untyped
+      def reduce(production_id)
+        production = parser_tables.fetch(:productions).fetch(production_id)
+        length = production.fetch(:length)
+        values = @value_stack.last(length)
+        hook_values = values.dup
+        @state_stack.pop(length)
+        @value_stack.pop(length)
+        result = reduction_value(production, values)
+        next_state = table_lookup(parser_tables.fetch(:gotos), @state_stack.last, production.fetch(:lhs))
+        raise ParseError, "(tables):1:1: missing goto for production #{production_id}" if next_state.nil?
+
+        @state_stack << next_state
+        @value_stack << result
+        trace("reduce #{production_id} (#{length}) -> state #{next_state}")
+        on_reduce(production_id, hook_values, result)
+        return [:done, result] if @accept_requested
+        return recover(report: false) if @semantic_error
+
+        [:continue]
+      end
+
+      # @rbs (Hash[Symbol, untyped] production, Array[untyped] values) -> untyped
+      def reduction_value(production, values)
+        action = production[:action]
+        return values.first unless action
+        return instance_exec(values, @value_stack.dup, &action) if action.respond_to?(:call)
+
+        __send__(action, values, @value_stack.dup)
+      end
+
+      # @rbs (?report: bool) -> untyped
+      def recover(report: true)
+        @semantic_error = false
+        if @recovery_shifts.positive?
+          return [:done, nil] if @lookahead == EOF_TOKEN
+
+          trace("discard #{token_to_str(@lookahead)} during recovery")
+          @lookahead = NO_LOOKAHEAD
+          return [:continue]
+        end
+
+        token_id = @lookahead
+        value = @lookahead_value
+        value_stack = @value_stack.dup
+        on_error(token_id, value, value_stack.dup) if report
+        return [:done, nil] unless shift_error_token
+
+        @recovery_shifts = RECOVERY_SHIFTS
+        on_error_recover(token_id, value, value_stack)
+        [:continue]
+      end
+
+      # @rbs () -> bool
+      def shift_error_token
+        loop do
+          action = table_lookup(parser_tables.fetch(:actions), @state_stack.last, ERROR_TOKEN)
+          if action&.first == :shift
+            trace("recover: shift error -> state #{action.fetch(1)}")
+            @state_stack << action.fetch(1)
+            @value_stack << nil
+            return true
+          end
+          return false if @state_stack.length == 1
+
+          trace("recover: pop state #{@state_stack.last}")
+          @state_stack.pop
+          @value_stack.pop
+        end
+      end
+
+      # @rbs () -> void
+      def read_lookahead
+        pair = read_external_token
+        if pair.nil? || pair == false
+          @lookahead = EOF_TOKEN
+          @lookahead_value = nil
+        else
+          external_token, @lookahead_value = pair
+          @lookahead = internal_token_id(external_token)
+        end
+        trace("read #{token_to_str(@lookahead)}")
+      end
+
+      # @rbs () -> untyped
+      def read_external_token
+        source = @source
+        raise ParseError, "(input):1:1: token source is not available" unless source
+
+        source.call
+      rescue StopIteration
+        false
+      end
+
+      # @rbs (untyped external_token) -> Integer
+      def internal_token_id(external_token)
+        token_id = parser_tables.fetch(:tokens)[external_token]
+        return token_id if token_id
+
+        @unknown_token_name = external_token.inspect
+        @unknown_token_id = -external_token.object_id.abs
+      end
+
+      # @rbs () -> Hash[Symbol, untyped]
+      def parser_tables
+        self.class.__send__(:parser_tables)
+      rescue NoMethodError
+        raise ParseError, "(tables):1:1: #{self.class} must define .parser_tables"
+      end
+
+      # @rbs (untyped action) -> bool
+      def error_action?(action)
+        action.nil? || action.first == :error
+      end
+
+      # @rbs (Integer state) -> untyped
+      def default_action(state)
+        parser_tables.fetch(:default_actions, EMPTY_ROW)[state]
+      end
+
+      # @rbs (untyped table, Integer row, Integer column) -> untyped
+      def table_lookup(table, row, column)
+        return table.lookup(row, column) if table.respond_to?(:lookup)
+
+        table.fetch(row, EMPTY_ROW)[column]
+      end
+
+      # @rbs (String message) -> void
+      def trace(message)
+        @yydebug_output.puts("ibex: #{message}") if @yydebug
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/ibex/runtime.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "runtime/parser"
+
+module Ibex
+  module Runtime
+  end
+end

data/lib/ibex/tables.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Parser table construction and row-displacement compression.
+  module Tables
+    # @rbs!
+    #   private def runtime_action: (IR::parser_action action) -> IR::runtime_action
+    #   private def self.runtime_action: (IR::parser_action action) -> IR::runtime_action
+
+    TableSet = Struct.new(:actions, :gotos, :default_actions, keyword_init: true)
+
+    # Sparse table represented by per-row offsets and ownership checks.
+    class Compact
+      attr_reader :offsets #: Array[Integer]
+      attr_reader :values #: Array[untyped]
+      attr_reader :checks #: Array[Integer?]
+      attr_reader :row_count #: Integer
+
+      class << self
+        # @rbs (Array[Hash[Integer, untyped]] rows) -> Compact
+        def build(rows)
+          offsets = Array.new(rows.length, 0)
+          values = [] #: Array[untyped]
+          checks = [] #: Array[Integer?]
+          rows.each_index.sort_by { |row| [-rows[row].length, row] }.each do |row|
+            offset = find_offset(rows[row].keys, checks)
+            offsets[row] = offset
+            rows[row].each do |column, value|
+              index = offset + column
+              values[index] = value
+              checks[index] = row
+            end
+          end
+          new(offsets: offsets, values: values, checks: checks, row_count: rows.length)
+        end
+
+        private
+
+        # @rbs (Array[Integer] columns, Array[Integer?] checks) -> Integer
+        def find_offset(columns, checks)
+          offset = 0
+          offset += 1 while columns.any? { |column| checks[offset + column] }
+          offset
+        end
+      end
+
+      # @rbs (offsets: Array[Integer], values: Array[untyped], checks: Array[Integer?], row_count: Integer) -> void
+      def initialize(offsets:, values:, checks:, row_count:)
+        @offsets = offsets.freeze
+        @values = values.freeze
+        @checks = checks.freeze
+        @row_count = row_count
+        freeze
+      end
+
+      # @rbs (Integer row, Integer column) -> untyped
+      def lookup(row, column)
+        return nil unless row.between?(0, @row_count - 1)
+        return nil if column.negative?
+
+        index = @offsets.fetch(row) + column
+        return nil unless index.between?(0, @checks.length - 1)
+
+        @checks[index] == row ? @values[index] : nil
+      end
+
+      # @rbs (Integer row) -> Hash[Integer, untyped]
+      def row(row)
+        return {} unless row.between?(0, @row_count - 1)
+
+        result = {} #: Hash[Integer, untyped]
+        @checks.each_index do |index|
+          next unless @checks[index] == row
+
+          result[index - @offsets[row]] = @values[index]
+        end
+        result
+      end
+    end
+
+    # @rbs (IR::Automaton automaton, ?format: Symbol | String) -> TableSet
+    def build(automaton, format: :compact)
+      action_rows = automaton.states.map do |state|
+        state.actions.transform_values do |action|
+          runtime_action(action)
+        end
+      end
+      goto_rows = automaton.states.map(&:gotos)
+      defaults = automaton.states.map do |state|
+        runtime_action(state.default_action) if state.default_action
+      end
+      if format.to_sym == :plain
+        return TableSet.new(actions: action_rows, gotos: goto_rows,
+                            default_actions: defaults)
+      end
+      raise ArgumentError, "unknown table format #{format.inspect}" unless format.to_sym == :compact
+
+      TableSet.new(actions: Compact.build(action_rows), gotos: Compact.build(goto_rows), default_actions: defaults)
+    end
+    module_function :build
+
+    # @rbs skip
+    private
+
+    # @rbs skip
+    def runtime_action(action)
+      case action.fetch(:type).to_sym
+      when :shift
+        shift = action #: IR::shift_action
+        [:shift, shift.fetch(:state)]
+      when :reduce
+        reduce = action #: IR::reduce_action
+        [:reduce, reduce.fetch(:production)]
+      when :accept then [:accept]
+      when :error then [:error]
+      else raise Ibex::Error, "unknown parser action #{action.inspect}"
+      end
+    end
+    module_function :runtime_action
+
+    class << self
+      private :runtime_action
+    end
+  end
+end

data/lib/ibex/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Ibex generates and runs Pure Ruby LR parsers.
+module Ibex
+  VERSION = "0.1.0"
+end

data/lib/ibex.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "ibex/version"
+require_relative "ibex/error"
+require_relative "ibex/tables"
+require_relative "ibex/runtime"
+require_relative "ibex/frontend"
+require_relative "ibex/ir"
+require_relative "ibex/normalize"
+require_relative "ibex/analysis"
+require_relative "ibex/lalr"
+require_relative "ibex/codegen/symbol_labels"
+require_relative "ibex/codegen/report"
+require_relative "ibex/codegen/ruby"
+require_relative "ibex/codegen/rbs"
+require_relative "ibex/codegen/dot"
+require_relative "ibex/codegen/html"
+
+# Ibex generates and runs Pure Ruby LR parsers.
+module Ibex
+  ParseError = Runtime::ParseError #: singleton(Runtime::ParseError)
+end

data/sig/ibex/analysis/sets.rbs

@@ -0,0 +1,72 @@
+# Generated from lib/ibex/analysis/sets.rb with RBS::Inline
+
+module Ibex
+  module Analysis
+    # Computes nullable, FIRST, and FOLLOW sets over Grammar IR using integer bitsets.
+    class Sets
+      attr_reader nullable_bits: Integer
+
+      attr_reader first_bits: Array[Integer]
+
+      attr_reader follow_bits: Array[Integer]
+
+      @grammar: IR::Grammar
+
+      # @rbs (IR::Grammar grammar) -> void
+      def initialize: (IR::Grammar grammar) -> void
+
+      # @rbs (Integer | String | Symbol | IR::GrammarSymbol symbol) -> bool
+      def nullable?: (Integer | String | Symbol | IR::GrammarSymbol symbol) -> bool
+
+      # @rbs (Integer | String | Symbol | IR::GrammarSymbol symbol) -> Array[String]
+      def first: (Integer | String | Symbol | IR::GrammarSymbol symbol) -> Array[String]
+
+      # @rbs (Integer | String | Symbol | IR::GrammarSymbol symbol) -> Array[String]
+      def follow: (Integer | String | Symbol | IR::GrammarSymbol symbol) -> Array[String]
+
+      # @rbs (Array[Integer] symbol_ids) -> Integer
+      def first_of_sequence: (Array[Integer] symbol_ids) -> Integer
+
+      # @rbs (Array[Integer] symbol_ids) -> bool
+      def sequence_nullable?: (Array[Integer] symbol_ids) -> bool
+
+      private
+
+      # @rbs () -> void
+      def compute_nullable: () -> void
+
+      # @rbs () -> [Array[Array[IR::Production]], Array[Integer], Array[Integer]]
+      def nullable_worklist: () -> [ Array[Array[IR::Production]], Array[Integer], Array[Integer] ]
+
+      # @rbs () -> void
+      def compute_first: () -> void
+
+      # @rbs () -> void
+      def compute_follow: () -> void
+
+      # @rbs (IR::Production production, Array[Array[Integer]] dependencies) -> void
+      def initialize_follow: (IR::Production production, Array[Array[Integer]] dependencies) -> void
+
+      # @rbs (Array[Integer] sets, Array[Array[Integer]] dependencies, Array[Integer] seeds) -> void
+      def propagate_bits: (Array[Integer] sets, Array[Array[Integer]] dependencies, Array[Integer] seeds) -> void
+
+      # @rbs (Integer id) -> bool
+      def nullable_id?: (Integer id) -> bool
+
+      # @rbs (Integer bits) -> Array[String]
+      def terminal_names: (Integer bits) -> Array[String]
+
+      # @rbs (Integer | String | Symbol | IR::GrammarSymbol symbol) -> Integer
+      def symbol_id: (Integer | String | Symbol | IR::GrammarSymbol symbol) -> Integer
+
+      # @rbs (Integer | String | Symbol | IR::GrammarSymbol symbol) -> IR::GrammarSymbol
+      def definition_for: (Integer | String | Symbol | IR::GrammarSymbol symbol) -> IR::GrammarSymbol
+
+      # @rbs (Integer id) -> IR::GrammarSymbol
+      def required_symbol_by_id: (Integer id) -> IR::GrammarSymbol
+
+      # @rbs (Integer id) -> Integer
+      def bit: (Integer id) -> Integer
+    end
+  end
+end

data/sig/ibex/analysis.rbs

@@ -0,0 +1,6 @@
+# Generated from lib/ibex/analysis.rb with RBS::Inline
+
+module Ibex
+  module Analysis
+  end
+end

data/sig/ibex/cli/counterexample_options.rbs

@@ -0,0 +1,16 @@
+# Generated from lib/ibex/cli/counterexample_options.rb with RBS::Inline
+
+module Ibex
+  # Counterexample report limits and their command-line validation.
+  module CLICounterexampleOptions
+    DEFAULTS: Hash[Symbol, Integer]
+
+    private
+
+    # @rbs (OptionParser options) -> void
+    def add_counterexample_options: (OptionParser options) -> void
+
+    # @rbs (Integer value, String option) -> Integer
+    def positive_counterexample_limit: (Integer value, String option) -> Integer
+  end
+end

data/sig/ibex/cli/outputs.rbs

@@ -0,0 +1,31 @@
+# Generated from lib/ibex/cli/outputs.rb with RBS::Inline
+
+module Ibex
+  # Files, warnings, and progress emitted by CLI pipeline stages.
+  module CLIOutputs
+    WARNING_MESSAGES: Hash[Symbol, ^(IR::grammar_warning) -> String]
+
+    private
+
+    # @rbs (String value) -> Array[Symbol]
+    def warning_categories: (String value) -> Array[Symbol]
+
+    # @rbs (IR::Grammar grammar, String input_path) -> void
+    def handle_grammar_warnings: (IR::Grammar grammar, String input_path) -> void
+
+    # @rbs (IR::grammar_warning warning, String input_path) -> String
+    def format_grammar_warning: (IR::grammar_warning warning, String input_path) -> String
+
+    # @rbs (IR::Automaton automaton, String input_path) -> void
+    def report_conflicts: (IR::Automaton automaton, String input_path) -> void
+
+    # @rbs (IR::Automaton automaton, String input_path) -> void
+    def write_report: (IR::Automaton automaton, String input_path) -> void
+
+    # @rbs (String input_path, String extension) -> String
+    def default_output_path: (String input_path, String extension) -> String
+
+    # @rbs (String message) -> void
+    def report_status: (String message) -> void
+  end
+end