rprec 1.0.0

data/lib/rprec/prec.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module RPrec
+  # `Prec` is an operator precedence.
+  class Prec
+    # @param name [Symbol]
+    # @param succs [Array<Symbol>]
+    # @param ops [Array<Op>]
+    def initialize(name, succs, ops)
+      @name = name
+      @succs = succs
+      @ops = ops
+    end
+
+    # @dynamic name, succs, ops
+
+    # @return [Symbol]
+    attr_reader :name
+    # @return [Array<Symbol>]
+    attr_reader :succs
+    # @return [Array<RPrec::Op>]
+    attr_reader :ops
+
+    # @api private
+    # @return [Hash{String => RPrec::Op}]
+    attr_reader :prefix_table
+    # @api private
+    # @return [Hash{String => RPrec::Op}]
+    attr_reader :postfix_table
+
+    # @api private
+    # @param grammar [RPrec::Grammar]
+    # @return [void]
+    def setup(grammar)
+      return unless @prefix_table.nil? && @postfix_table.nil?
+
+      @prefix_table = {}
+      @postfix_table = {}
+
+      @ops.each do |op|
+        case op.type
+        when :prefix, :non_assoc_prefix, :closed
+          raise ArgumentError, "Conflict with the key token '#{op.key}'" if @prefix_table.include?(op.key)
+
+          @prefix_table[op.key] = op
+        when :postfix, :non_assoc_postfix, :left_assoc, :right_assoc, :non_assoc
+          raise ArgumentError, "Conflict with the key token '#{op.key}'" if @postfix_table.include?(op.key)
+
+          @postfix_table[op.key] = op
+        end
+      end
+
+      grammar.setup(@succs)
+
+      @ops.each do |op|
+        parts = op.parts.filter_map { _1.is_a?(Symbol) ? _1 : nil }
+        grammar.setup(parts)
+      end
+
+      @all_prefix_keys = @prefix_table.keys
+      @all_postfix_keys = @postfix_table.keys
+      @prefix_keys = @prefix_table.filter { |_, op| op.type == :prefix }.keys
+      @right_assoc_keys = @postfix_table.filter { |_, op| op.type == :right_assoc }.keys
+      @postfix_and_left_assoc_keys = @postfix_table.filter { |_, op| op.type == :postfix || op.type == :left_assoc }.keys
+    end
+
+    # @api private
+    # @param grammar [RPrec::Grammar]
+    # @param stream [RPrec::Stream]
+    # @return [Object, nil]
+    def parse(grammar, stream)
+      op = @prefix_table[stream.current.type]
+
+      if op
+        key_token = stream.current
+        stream.next
+
+        case op.type
+        when :prefix
+          parts = grammar.parse_parts(op.parts, stream)
+          return parse_prefix_and_right_assoc(grammar, stream, [[op, [key_token, *parts]]])
+
+        when :non_assoc_prefix
+          parts = grammar.parse_parts(op.parts, stream)
+          node = grammar.parse_precs(@succs, stream)
+          return op.build(key_token, *parts, node)
+
+        when :closed
+          parts = grammar.parse_parts(op.parts, stream)
+          return op.build(key_token, *parts)
+
+        else raise ScriptError, 'Unreachable'
+        end
+      end
+      stream.expected(@all_prefix_keys)
+
+      node = grammar.parse_precs(@succs, stream)
+      key_token = stream.current
+      op = @postfix_table[key_token.type]
+      unless op
+        stream.expected(@all_postfix_keys)
+        return node
+      end
+      stream.next
+
+      case op.type
+      when :postfix
+        parts = grammar.parse_parts(op.parts, stream)
+        node = op.build(node, key_token, *parts)
+        parse_postfix_and_left_assoc(grammar, stream, node)
+
+      when :non_assoc_postfix
+        parts = grammar.parse_parts(op.parts, stream)
+        op.build(node, key_token, *parts)
+
+      when :left_assoc
+        parts = grammar.parse_parts(op.parts, stream)
+        right = grammar.parse_precs(@succs, stream)
+        node = op.build(node, key_token, *parts, right)
+        parse_postfix_and_left_assoc(grammar, stream, node)
+
+      when :right_assoc
+        parts = grammar.parse_parts(op.parts, stream)
+        parse_prefix_and_right_assoc(grammar, stream, [[op, [node, key_token, *parts]]])
+
+      when :non_assoc
+        parts = grammar.parse_parts(op.parts, stream)
+        right = grammar.parse_precs(@succs, stream)
+        op.build(node, key_token, *parts, right)
+
+      else raise ScriptError, 'Unreachable'
+      end
+    end
+
+    # @param grammar [RPrec::Grammar]
+    # @param stream [RPrec::Stream]
+    # @param stack [Array<Array<RPrec::Op, RPrec::Token, Object>>]
+    # @return [Object]
+    private def parse_prefix_and_right_assoc(grammar, stream, stack)
+      continue = true
+      # TODO: this untyped var seems ugly. How to fix it?
+      # @type var node: untyped
+      node = nil
+      while continue
+        key_token = stream.current
+        op = @prefix_table[key_token.type]
+        while op.is_a?(Op) && op.type == :prefix
+          stream.next
+          parts = grammar.parse_parts(op.parts, stream)
+          stack << [op, [key_token, *parts]]
+          key_token = stream.current
+          op = @prefix_table[key_token.type]
+        end
+        stream.expected(@prefix_keys)
+
+        node = grammar.parse_precs(@succs, stream)
+        key_token = stream.current
+        op = @postfix_table[key_token.type]
+        if op.is_a?(Op) && op.type == :right_assoc
+          stream.next
+          parts = grammar.parse_parts(op.parts, stream)
+          stack << [op, [node, key_token, *parts]]
+          continue = true
+        else
+          stream.expected(@right_assoc_keys)
+          continue = false
+        end
+      end
+
+      stack.reverse_each do |(parsed_op, args)|
+        node = parsed_op.build(*args, node)
+      end
+      node
+    end
+
+    # @param grammar [RPrec::Grammar]
+    # @param stream [RPrec::Stream]
+    # @param node [Object]
+    # @return [Object]
+    private def parse_postfix_and_left_assoc(grammar, stream, node)
+      key_token = stream.current
+      op = @postfix_table[key_token.type]
+      while op.is_a?(Op) && (op.type == :postfix || op.type == :left_assoc)
+        stream.next
+        case op.type
+        when :postfix
+          parts = grammar.parse_parts(op.parts, stream)
+          node = op.build(node, key_token, *parts)
+        when :left_assoc
+          parts = grammar.parse_parts(op.parts, stream)
+          right = grammar.parse_precs(@succs, stream)
+          node = op.build(node, key_token, *parts, right)
+        end
+        key_token = stream.current
+        op = @postfix_table[key_token.type]
+      end
+      stream.expected(@postfix_and_left_assoc_keys)
+      node
+    end
+
+    # @return [String]
+    def inspect
+      result = "prec #{name.inspect}"
+      result << " => #{succs.inspect}" unless succs.empty?
+      unless ops.empty?
+        result << " do\n"
+        ops.each do |op|
+          result << "  #{op.inspect}\n"
+        end
+        result << 'end'
+      end
+      result
+    end
+  end
+end

data/lib/rprec/regexp_lexer.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative 'stream'
+
+module RPrec
+  # `RegexpLexer` is a lexer using regexp patterns.
+  class RegexpLexer
+    # @param skip [Regexp]
+    # @param pattern [Regexp]
+    # @param block [Proc]
+    def initialize(skip:, pattern:, &block)
+      @skip = /\G#{skip}/
+      @pattern = /\G#{pattern}/
+      @block = block
+    end
+
+    # @dynamic skip, pattern, block
+
+    # @return [Regexp]
+    attr_reader :skip
+    # @return [Regexp]
+    attr_reader :pattern
+    # @return [Proc]
+    attr_reader :block
+
+    # @param source [String]
+    # @return [RPrec::RegexpStream]
+    def lex(source)
+      RegexpStream.new(self, source)
+    end
+  end
+
+  # `RegexpStream` is a stream generated by a `RegexpLexer`.
+  #
+  # @api private
+  class RegexpStream < Stream
+    # @param lexer [RPrec::Lexer]
+    # @param source [String]
+    def initialize(lexer, source)
+      super()
+      @lexer = lexer
+      @source = source
+      @offset = 0
+      self.next
+    end
+
+    # @return [RPrec::Token]
+    attr_reader :current
+
+    # @return [void]
+    def next
+      super
+
+      skip = @source.match(@lexer.skip, @offset)
+      @offset += skip.to_s.size if skip
+
+      if eof?
+        @current = Token.new('EOF', loc: @offset...@offset)
+        return
+      end
+
+      match = @source.match(@lexer.pattern, @offset)
+      raise ParseError.new("Unexpected character: #{@source[@offset].inspect}", loc: @offset...@offset + 1) unless match
+
+      type_and_value = @lexer.block.call(match)
+      type, value =
+        if type_and_value.is_a?(Array)
+          [type_and_value[0], type_and_value[1]]
+        else
+          [type_and_value, nil]
+        end
+      loc = @offset...@offset + match.to_s.size
+      @offset += match.to_s.size
+      @current = Token.new(type, value, loc:)
+    end
+
+    # @return [Boolean]
+    def eof?
+      @source.size <= @offset
+    end
+  end
+end

data/lib/rprec/stream.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module RPrec
+  # `Stream` is a token stream.
+  #
+  # @abstract `current`, `next` and `eof?` must be implemented.
+  class Stream
+    def initialize
+      @expected = []
+    end
+
+    # @return [RPrec::Token]
+    def current
+      raise ScriptError, 'Not implemented'
+    end
+
+    # @return [void]
+    def next
+      @expected = []
+    end
+
+    # @return [Boolean]
+    def eof?
+      raise ScriptError, 'Not implemented'
+    end
+
+    # @param tokens [Array<String>]
+    # @return [void]
+    def expected(tokens)
+      @expected += tokens
+    end
+
+    # @return [void]
+    def unexpected
+      token = current
+      raise ParseError.new("Unexpected token '#{token.type}'", loc: token.loc) if @expected.empty?
+
+      expected = @expected.sort.map { |tok| "'#{tok}'" }.join(', ')
+      raise ParseError.new("Expected token(s) #{expected}, but the unexpected token '#{token.type}' comes", loc: token.loc)
+    end
+  end
+end

data/lib/rprec/token.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module RPrec
+  # `Token` is a token.
+  class Token
+    # @param type [String]
+    # @param value [Object]
+    # @param loc [Range, nil]
+    def initialize(type, value = nil, loc: nil)
+      @type = type
+      @value = value
+      @loc = loc
+    end
+
+    # @dynamic type, value, loc
+
+    # @return [String]
+    attr_reader :type
+    # @return [Object]
+    attr_reader :value
+    # @return [Range, nil]
+    attr_reader :loc
+  end
+end

data/lib/rprec/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module RPrec
+  # The version.
+  VERSION = '1.0.0'
+end

data/lib/rprec.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+# `RPrec` is an implementation of operator-precedence parsing.
+# The operator-precedence parsing is also known as Pratt parsing.
+# This implementation is extended for mixfix operators which are operators
+# consists of multi parts (e.g. `... ? ... : ...` operator).
+#
+# ## Usage
+#
+# `RPrec::Grammar` is a grammar of the target language and also behaves
+# its parser.
+#
+# We have a `RPrec::DSL` to easily build a `RPrec::Grammar` instance.
+# The below is an example grammar for simple arithmetic expressions.
+#
+# ```
+# grammar = RPrec::DSL.build do
+#   prec :main => :add_sub
+#
+#   prec :add_sub => :mul_div do
+#     left_assoc '+' do |left, op_tok, right|
+#       [:add, left, right]
+#     end
+#     left_assoc '-' do |left, op_tok, right|
+#       [:sub, left, right]
+#     end
+#   end
+#
+#   prec :mul_div => :unary do
+#     left_assoc '*' do |left, op_tok, right|
+#       [:mul, left, right]
+#     end
+#     left_assoc '/' do |left, op_tok, right|
+#       [:div, left, right]
+#     end
+#   end
+#
+#   prec :unary => :atom do
+#     prefix '+' do |op_tok, expr|
+#       [:plus, expr]
+#     end
+#     prefix '-' do |op_tok, expr|
+#       [:minus, expr]
+#     end
+#   end
+#
+#   prec :atom do
+#     closed 'nat' do |nat_tok|
+#       [:nat, nat_tok.value]
+#     end
+#     closed '(', :add_sub, ')' do |lpar_tok, expr, rpar_tok|
+#       [:paren, expr]
+#     end
+#   end
+# end
+# ```
+#
+# Here, `prec` is the only available method in the `RPrec::DSL` context.
+# This method is for defining a precedence and takes the argument of one of the
+# following forms: `name => succs`, `name => succs`, and `name`.
+# `prec name => succ` is an alias for `prec name => [succ]` and `prec name` is
+# an alias for `prec name => []`. `succs` is an array of precedence names which
+# have higher binding powers than this precedence's. Precedence names must be
+# symbols.
+#
+# `prec` also can take a block to define operators belonging to this
+# precedence. Operators can be defined by the following eight methods:
+#
+#   - `prefix key, *parts, &block`
+#   - `postfix key, *parts, &block`
+#   - `non_assoc_prefix key, *parts, &block`
+#   - `non_assoc_postfix key, *parts, &block`
+#   - `closed key, *parts, &block`
+#   - `left_assoc key, *parts, &block`
+#   - `right_assoc key, *parts, &block`
+#   - `non_assoc key, *parts, &block`
+#
+# They define operators with their own name associativity. The first argument
+# `key` specifies the key token that is used to determine which the operator to
+# be parsed. The rest arguments are `parts`: they consist of precedence names
+# or tokens, meaning that they are required after the key token. They can take
+# a block, which is an action called on parsed.
+#
+# To parse a string using this grammar, we can use the `RPrec::Grammar#parse`
+# method. This method takes a `RPrec::Stream` object. `RPrec::Stream`
+# is a token stream (an array) with convenient methods for parsing. It can be
+# created easily by using `RPrec::ArrayStream` class.
+#
+# ```
+# # A stream for the source `"1 + 2 * 3"`.
+# stream = RPrec::ArrayStream.new([
+#   RPrec::Token.new('nat', 1),
+#   RPrec::Token.new('+'),
+#   RPrec::Token.new('nat', 2),
+#   RPrec::Token.new('*'),
+#   RPrec::Token.new('nat', 3),
+# ])
+#
+# grammar.parse(stream)
+# # => [:add, [:nat, 1], [:mul, [:nat, 2], [:nat, 3]]]
+# ```
+#
+# ## Limitations
+#
+# This implementation assumes that an operator can be determined by the current
+# token on parsing. If a grammar is not so, this reports the "Conflict with the
+# key token ..." error.
+#
+# For example, the following grammar is rejected because the key token `'['`
+# is conflicted.
+#
+# ```
+# grammar = RPrec::DSL.build do
+#   prec :range do
+#      closed '[', :int, ',', :int, ']'
+#      closed '[', :int, ',', :int, ')'
+#   end
+# end
+# # raises "Conflict with the key token '['"
+# ```
+#
+# ## References
+#
+#   - [Operator-precedence parser - Wikipedia](https://en.wikipedia.org/wiki/Operator-precedence_parser)
+#   - [Parsing Mixfix Operators | Springer Link](https://link.springer.com/chapter/10.1007/978-3-642-24452-0_5)
+module RPrec
+  # `Error` is an error for `RPrec`.
+  class Error < StandardError
+  end
+end
+
+require_relative 'rprec/array_stream'
+require_relative 'rprec/dsl'
+require_relative 'rprec/grammar'
+require_relative 'rprec/op'
+require_relative 'rprec/parse_error'
+require_relative 'rprec/prec'
+require_relative 'rprec/regexp_lexer'
+require_relative 'rprec/stream'
+require_relative 'rprec/token'
+require_relative 'rprec/version'

data/rbs_collection.lock.yaml

@@ -0,0 +1,12 @@
+---
+path: ".gem_rbs_collection"
+gems:
+- name: minitest
+  version: '0'
+  source:
+    type: stdlib
+- name: mutex_m
+  version: '0'
+  source:
+    type: stdlib
+gemfile_lock_path: Gemfile.lock

data/rbs_collection.yaml

@@ -0,0 +1,20 @@
+sources:
+  - type: git
+    name: ruby/gem_rbs_collection
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    revision: main
+    repo_dir: gems
+
+path: .gem_rbs_collection
+
+gems:
+  - name: rake
+    ignore: true
+  - name: rbs
+    ignore: true
+  - name: redcarpet
+    ignore: true
+  - name: steep
+    ignore: true
+  - name: yard
+    ignore: true

data/sig/rprec/array_stream.rbs

@@ -0,0 +1,7 @@
+module RPrec
+  class ArrayStream[T] < Stream[T]
+    def initialize: (Array[Token[T]]) -> void
+    @tokens: Array[Token[T]]
+    @index: Integer
+  end
+end

data/sig/rprec/dsl.rbs

@@ -0,0 +1,26 @@
+module RPrec
+  class DSL[T, R < Object]
+    def self.build: [T, R < Object] (?Symbol) { (DSL[T, R]) [self: DSL[T, R]] -> void } -> Grammar[T, R]
+    def initialize: () -> void
+    attr_reader precs: Hash[Symbol, Prec[T, R]]
+    def prec:
+      (Symbol | Hash[Symbol, Symbol | Array[Symbol]]) ?{ (PrecDSL[T, R]) [self: PrecDSL[T, R]] -> void }
+      -> void
+
+    class PrecDSL[T, R < Object]
+      def self.build:
+        [T, R < Object] (Symbol, Array[Symbol]) ?{ (PrecDSL[T, R]) [self: PrecDSL[T, R]] -> void }
+        -> Prec[T, R]
+      def initialize: () -> void
+      attr_reader ops: Array[Op[T, R]]
+      def prefix: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def postfix: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def non_assoc_prefix: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def non_assoc_postfix: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def closed: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def left_assoc: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def right_assoc: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+      def non_assoc: (String, *(String | Symbol)) { (*(Token[T] | R)) -> R } -> void
+    end
+  end
+end

data/sig/rprec/grammar.rbs

@@ -0,0 +1,13 @@
+module RPrec
+  class Grammar[T, R < Object]
+    def initialize: (Symbol, Hash[Symbol, Prec[T, R]]) -> void
+    def []: (Symbol) -> Prec[T, R]
+    def setup: (?Array[Symbol]) -> void
+    def parse: (Stream[T], ?check_eof: bool) -> R
+    def parse_prec: (Symbol, Stream[T]) -> R?
+    def parse_precs: (Array[Symbol], Stream[T]) -> R
+    def parse_parts: (Array[String | Symbol], Stream[T]) -> Array[Token[T] | R]
+    @main: Symbol
+    @precs: Hash[Symbol, Prec[T, R]]
+  end
+end

data/sig/rprec/op.rbs

@@ -0,0 +1,15 @@
+module RPrec
+  type op_type =
+    :prefix | :postfix | :non_assoc_prefix | :non_assoc_postfix | :closed |
+    :left_assoc | :right_assoc | :non_assoc
+
+  class Op[T, R < Object]
+    def initialize:
+      (op_type, String, Array[String | Symbol], ^(*(Token[T] | R)) -> R) -> void
+    attr_reader type: op_type
+    attr_reader key: String
+    attr_reader parts: Array[String | Symbol]
+    def build: (*(Token[T] | R)) -> R
+    @build: ^(*(Token[T] | R)) -> R
+  end
+end

data/sig/rprec/parse_error.rbs

@@ -0,0 +1,6 @@
+module RPrec
+  class ParseError < Error
+    def initialize: (String, ?loc: Range[Integer]?) -> void
+    attr_reader loc: Range[Integer]?
+  end
+end

data/sig/rprec/prec.rbs

@@ -0,0 +1,21 @@
+module RPrec
+  class Prec[T, R < Object]
+    def initialize: (Symbol, Array[Symbol], Array[Op[T, R]]) -> void
+    attr_reader name: Symbol
+    attr_reader succs: Array[Symbol]
+    attr_reader ops: Array[Op[T, R]]
+    def setup: (Grammar[T, R]) -> void
+    def parse: (Grammar[T, R], Stream[T]) -> R?
+    private def parse_prefix_and_right_assoc:
+      (Grammar[T, R], Stream[T], Array[[Op[T, R], Array[Token[T] | R]]]) -> R
+    private def parse_postfix_and_left_assoc:
+      (Grammar[T, R], Stream[T], R) -> R
+    @prefix_table: Hash[String, Op[T, R]]
+    @postfix_table: Hash[String, Op[T, R]]
+    @all_prefix_keys: Array[String]
+    @all_postfix_keys: Array[String]
+    @prefix_keys: Array[String]
+    @right_assoc_keys: Array[String]
+    @postfix_and_left_assoc_keys: Array[String]
+  end
+end

data/sig/rprec/regexp_lexer.rbs

@@ -0,0 +1,19 @@
+module RPrec
+  class RegexpLexer[T]
+    def initialize:
+      (skip: Regexp, pattern: Regexp) { (MatchData) -> (String | [String, T]) }
+      -> void
+    def lex: (String) -> Stream[T]
+    attr_reader skip: Regexp
+    attr_reader pattern: Regexp
+    attr_reader block: ^(MatchData) -> (String | [String, T])
+  end
+
+  class RegexpStream[T] < Stream[T]
+    def initialize: (RegexpLexer[T], String) -> void
+    @lexer: RegexpLexer[T]
+    @source: String
+    @offset: Integer
+    @current: Token[T]
+  end
+end

data/sig/rprec/stream.rbs

@@ -0,0 +1,11 @@
+module RPrec
+  class Stream[T]
+    def initialize: () -> void
+    def current: () -> Token[T]
+    def next: () -> void
+    def eof?: () -> bool
+    def expected: (Array[String]) -> void
+    def unexpected: () -> bot
+    @expected: Array[String]
+  end
+end

data/sig/rprec/token.rbs

@@ -0,0 +1,9 @@
+module RPrec
+  class Token[T]
+    def initialize: (String, ?(T | nil), ?loc: Range[Integer]?) -> void
+                  
+    attr_reader type: String
+    attr_reader value: T?
+    attr_reader loc: Range[Integer]?
+  end
+end

data/sig/rprec/version.rbs

@@ -0,0 +1,3 @@
+module RPrec
+  VERSION: String
+end