pestle 0.1.0

data/examples/calculator_prec_climber.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+# Based on the calculator example found in the pest book, with
+# the addition of the `ident` rule.
+# https://pest.rs/book/precedence.html.
+
+# https://github.com/pest-parser/book/blob/master/LICENSE-MIT
+#
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without
+# limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software
+# is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+# ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+# IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require_relative "../lib/pestle"
+
+# Example calculator parser using a precedence climbing technique to handle operator precedence.
+module PrecClimberExample
+  GRAMMAR = <<~'GRAMMAR'
+    WHITESPACE   =  _{ " " | "\t" | NEWLINE }
+
+    program      =   { SOI ~ expr ~ EOI }
+      expr       =   { prefix* ~ primary ~ postfix* ~ (infix ~ prefix* ~ primary ~ postfix* )* }
+        infix    =  _{ add | sub | mul | div | pow }
+          add    =   { "+" } // Addition
+          sub    =   { "-" } // Subtraction
+          mul    =   { "*" } // Multiplication
+          div    =   { "/" } // Division
+          pow    =   { "^" } // Exponentiation
+        prefix   =  _{ neg }
+          neg    =   { "-" } // Negation
+        postfix  =  _{ fac }
+          fac    =   { "!" } // Factorial
+        primary  =  _{ int | "(" ~ expr ~ ")" | ident }
+          int    =  @{ (ASCII_NONZERO_DIGIT ~ ASCII_DIGIT+ | ASCII_DIGIT) }
+          ident  =  @{ ASCII_ALPHA+ }
+  GRAMMAR
+
+  PARSER = Pestle::Parser.from_grammar(GRAMMAR)
+
+  START_RULE = :program
+
+  # Very basic abstract syntax tree (AST) nodes.
+
+  VarExpr = Struct.new(:value) do
+    def evaluate(vars) = vars[value]
+  end
+
+  IntExpr = Struct.new(:value) do
+    def evaluate(vars) = value # rubocop: disable Lint/UnusedMethodArgument
+  end
+
+  PrefixExpr = Struct.new(:op, :expr) do
+    def evaluate(vars) = expr.evaluate(vars).send(op)
+  end
+
+  InfixExpr = Struct.new(:left, :op, :right) do
+    def evaluate(vars) = left.evaluate(vars).send(op, right.evaluate(vars))
+  end
+
+  PostfixExpr = Struct.new(:expr, :op) do
+    def evaluate(vars) = expr.evaluate(vars).send(op)
+  end
+
+  # Monkey patch Integer with a factorial method.
+  class ::Integer
+    remove_method(:fact) if method_defined?(:fact)
+    def fact
+      (1..self).reduce(1, :*)
+    end
+  end
+
+  # Operator precedence
+  PREC_LOWEST = 1
+  PREC_ADD = 3
+  PREC_SUB = 3
+  PREC_MUL = 4
+  PREC_DIV = 4
+  PREC_POW = 5
+  PREC_FAC = 7
+  PREC_PRE = 8
+
+  PRECEDENCES = {
+    add: PREC_ADD,
+    sub: PREC_SUB,
+    mul: PREC_MUL,
+    div: PREC_DIV,
+    pow: PREC_POW,
+    fac: PREC_FAC
+  }.freeze
+
+  INFIX_OPS = Set.new(%i[add sub mul div pow]).freeze
+
+  PREFIX_OPS = Set.new(%i[neg]).freeze
+
+  POSTFIX_OPS = Set.new(%i[fac]).freeze
+
+  # Calculator expression parser entry point.
+  # @param text [String]
+  # @return [Expr] A tree structure built from expression structs defined above.
+  def self.parse_program(text)
+    program = PARSER.parse(START_RULE, text)
+
+    # A successful parse is guaranteed to give us a root "program" pair with a
+    # single child. Unwrap it and get it's children as a Pestle::Pairs instance.
+    pairs = program.first.inner
+
+    # For this grammar, the `program` token is guaranteed to have exactly two
+    # children, an `expr` and `EOI`. Turn the expression into a stream and
+    # pass it to `parse_expr`.
+    parse_expr(pairs.first.stream)
+  end
+
+  # @param pairs [Pestle::Stream]
+  # @param precedence [Integer]
+  def self.parse_expr(pairs, precedence = PREC_LOWEST)
+    pair = pairs.next
+    raise "unexpected end of expression" if pair.nil?
+
+    left = if PREFIX_OPS.include?(pair.rule)
+             parse_prefix_expr(pair, pairs)
+           else
+             parse_basic_expr(pair)
+           end
+
+    pair = pairs.next
+
+    # Handle infix operators.
+    while !pair.nil? && INFIX_OPS.include?(pair.rule)
+      if (PRECEDENCES[pair.rule] || PREC_LOWEST) >= precedence
+        left = parse_infix_expr(pair, pairs, left)
+        pair = pairs.next
+      else
+        pairs.backup
+        return left
+      end
+    end
+
+    # Handle postfix operators
+    while !pair.nil? && POSTFIX_OPS.include?(pair.rule)
+      left = parse_postfix_expr(pair, left)
+      pair = pairs.next
+    end
+
+    raise "unexpected #{pair.text.inspect}" unless pair.nil? || pair.rule == :EOI
+
+    left
+  end
+
+  # @param pair [Pestle::Pair]
+  # @param pairs [Pestle::Stream]
+  def self.parse_prefix_expr(pair, pairs)
+    raise "unknown prefix operator #{pair.text.inspect}" unless PREFIX_OPS.include?(pair.rule)
+
+    PrefixExpr.new(:-@, parse_expr(pairs, PREC_PRE))
+  end
+
+  # @param op [Pestle::Pair]
+  # @param pairs [Pestle::Stream]
+  # @param left [Pestle::Pair]
+  def self.parse_infix_expr(op, pairs, left)
+    precedence = PRECEDENCES[op.rule] || PREC_LOWEST
+    right = parse_expr(pairs, precedence)
+
+    case op
+    in :add, _
+      InfixExpr.new(left, :+, right)
+    in :sub, _
+      InfixExpr.new(left, :-, right)
+    in :mul, _
+      InfixExpr.new(left, :*, right)
+    in :div, _
+      InfixExpr.new(left, :/, right)
+    in :pow, _
+      InfixExpr.new(left, :**, right)
+    else
+      raise "unknown infix operator #{op.text.inspect}"
+    end
+  end
+
+  # @param op [Pestle::Pair]
+  # @param left [Pestle::Pair]
+  def self.parse_postfix_expr(op, left)
+    raise "unknown postfix operator #{op.text.inspect}" unless POSTFIX_OPS.include?(op.rule)
+
+    PostfixExpr.new(left, :fact)
+  end
+
+  # @param pair [Pestle::Pair]
+  def self.parse_basic_expr(pair)
+    case pair
+    in :int, _
+      IntExpr.new(pair.text.to_i)
+    in :ident, _
+      VarExpr.new(pair.text)
+    in :expr, _
+      parse_expr(pair.inner.stream)
+    else
+      raise "unexpected #{pair.text.inspect}"
+    end
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  prog = PrecClimberExample.parse_program("1 + 2 + x")
+  puts prog.evaluate({ "x" => 39 })
+end

data/examples/calculator_stack_vm.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+# Just for fun, a bytecode compiler and stack-based virtual machine for
+# compiling and running expressions parsed with our Pratt parser example.
+
+require_relative "calculator_pratt"
+
+# Calculator expression parser, interpreter, compiler and virtual machine.
+module PrattExample
+  Bytecode = Data.define(:instructions, :constants, :symbols)
+
+  # Bytecode definition and helpers for encoding and decoding bytecode.
+  module Code
+    # 8-bit integer op codes
+    OP_CONSTANT = 1
+    OP_ADD = 2
+    OP_SUB = 3
+    OP_MUL = 4
+    OP_DIV = 5
+    OP_POW = 6
+    OP_NEG = 7
+    OP_FAC = 8
+    OP_VAR = 9
+
+    # @param name [String] Human readable name for the op code.
+    # @param operand_widths [Array[Integer]] A byte count for each operand.
+    OpDef = Data.define(:name, :operand_widths)
+
+    # @type const DEFS: Hash[Integer, OpDef]
+    DEFS = {
+      OP_CONSTANT => OpDef.new("OpConstant", [2]),
+      OP_VAR => OpDef.new("OpVar", [2]),
+      OP_ADD => OpDef.new("OpAdd", []),
+      OP_SUB => OpDef.new("OpSub", []),
+      OP_MUL => OpDef.new("OpMul", []),
+      OP_DIV => OpDef.new("OpDiv", []),
+      OP_POW => OpDef.new("OpPow", []),
+      OP_NEG => OpDef.new("OpNeg", []),
+      OP_FAC => OpDef.new("OpFac", [])
+    }.freeze
+
+    # Make a bytecode instruction for operator `op` with operands from `operands`.
+    # @param op [Integer] Op code.
+    # @param operands [*Integer] Operands for `op`.
+    # @return [Array[Integer]] Bytes for one instruction.
+    def self.make(op, *operands)
+      op_def = DEFS[op]
+      raise "unknown op code #{op}" if op_def.nil?
+
+      instruction = [op]
+
+      # Encode operands with big-endian byte order.
+      operands.zip(op_def.operand_widths) do |operand, byte_count|
+        (byte_count - 1).downto(0) do |byte_index|
+          instruction << ((operand >> (byte_index * 8)) & 0xFF)
+        end
+      end
+
+      instruction
+    end
+
+    # Read operands for the operator defined by `op_def` from `instructions`
+    # starting from `offset`.
+    # @param op_def [OpDef]
+    # @param instructions [Array[Integer]] Array of bytes.
+    # @param offset [Integer] Index into `instructions`.
+    # @return [[Array[Integer], Integer]] Operands read and new offset.
+    def self.read(op_def, instructions, offset)
+      operands = []
+
+      op_def.operand_widths.each do |byte_count|
+        value = 0
+        (0...byte_count).each do |i|
+          value = (value << 8) | instructions[offset + i]
+        end
+        operands << value
+        offset += byte_count
+      end
+
+      [operands, offset]
+    end
+
+    # Read `n_bytes` bytes as a single operand from `instructions` starting at `offset`.
+    def self.read_bytes(n_bytes, instructions, offset)
+      value = 0
+      (0...n_bytes).each { |i| value = (value << 8) | instructions[offset + i] }
+      value
+    end
+
+    def self.to_s(instructions)
+      buf = []
+      i = 0
+
+      while i < instructions.length
+        op_def = DEFS[instructions[i]]
+        raise "unknown op code #{instructions[i]}" if op_def.nil?
+
+        operands, new_offset = read(op_def, instructions, i + 1)
+
+        unless op_def.operand_widths.length == operands.length
+          raise "expected #{op_def.operand_widths.length} operands, got #{operands.length}"
+        end
+
+        buf << format("%04d %s %s", i, op_def.name, operands.map(&:to_s).join(" "))
+        i = new_offset
+      end
+
+      buf.join("\n")
+    end
+  end
+
+  # A simple calculator compiler.
+  class Compiler
+    def initialize
+      @instructions = []
+      @constants = []
+      @symbols = []
+    end
+
+    def compile(node)
+      case node
+      when IntExpr
+        emit(Code::OP_CONSTANT, add_constant(node.value))
+      when VarExpr
+        emit(Code::OP_VAR, add_symbol(node.value))
+      when InfixExpr
+        compile(node.left)
+        compile(node.right)
+        case node.op
+        when :+
+          emit(Code::OP_ADD)
+        when :-
+          emit(Code::OP_SUB)
+        when :*
+          emit(Code::OP_MUL)
+        when :/
+          emit(Code::OP_DIV)
+        when :**
+          emit(Code::OP_POW)
+        else
+          raise "unknown infix operator #{node.op}"
+        end
+      when PrefixExpr
+        compile(node.expr)
+        case node.op
+        when :-@
+          emit(Code::OP_NEG)
+        else
+          raise "unknown prefix operator #{node.op}"
+        end
+      when PostfixExpr
+        compile(node.expr)
+        case node.op
+        when :fact
+          emit(Code::OP_FAC)
+        else
+          raise "unknown postfix operator #{node.op}"
+        end
+      else
+        raise "unexpected expression #{node.class}"
+      end
+    end
+
+    def bytecode
+      Bytecode.new(@instructions, @constants, @symbols)
+    end
+
+    private
+
+    def add_constant(const)
+      @constants << const
+      @constants.length - 1
+    end
+
+    def add_symbol(name)
+      @symbols << name
+      @symbols.length - 1
+    end
+
+    def emit(op, *operands)
+      add_instruction(Code.make(op, *operands))
+    end
+
+    def add_instruction(instruction)
+      @instructions.concat(instruction)
+      @instructions.length - 1
+    end
+  end
+
+  # A virtual machine running our simple calculator bytecode.
+  class VM
+    def initialize(bytecode, vars)
+      @instructions = bytecode.instructions
+      @constants = bytecode.constants
+      @symbols = bytecode.symbols
+      @stack = Array.new(2048)
+      @sp = 0
+
+      @vars = vars
+    end
+
+    def stack_top = @stack[@sp - 1]
+
+    def run
+      ip = 0
+      loop do
+        break if ip >= @instructions.length
+
+        op = @instructions[ip]
+        case op
+        when Code::OP_CONSTANT
+          const_index = Code.read_bytes(2, @instructions, ip + 1)
+          ip += 3
+          push(@constants[const_index])
+        when Code::OP_VAR
+          symbol_index = Code.read_bytes(2, @instructions, ip + 1)
+          ip += 3
+
+          name = @symbols[symbol_index]
+          val = @vars[name]
+          raise "undefined variable #{name}" if val.nil?
+
+          push(val)
+        when Code::OP_ADD
+          right = pop
+          left = pop
+
+          push(left + right)
+          ip += 1
+        when Code::OP_SUB
+          right = pop
+          left = pop
+
+          push(left - right)
+          ip += 1
+        when Code::OP_MUL
+          right = pop
+          left = pop
+
+          push(left * right)
+          ip += 1
+        when Code::OP_DIV
+          right = pop
+          left = pop
+
+          push(left / right)
+          ip += 1
+        when Code::OP_POW
+          right = pop
+          left = pop
+
+          push(left**right)
+          ip += 1
+        when Code::OP_NEG
+          push(-pop)
+          ip += 1
+        when Code::OP_FAC
+          push(pop.fact)
+          ip += 1
+        end
+      end
+    end
+
+    private
+
+    def push(obj)
+      @stack[@sp] = obj
+      @sp += 1
+      nil
+    end
+
+    def pop
+      obj = @stack[@sp - 1]
+      @sp -= 1
+      obj
+    end
+  end
+
+  def self.compile(prog)
+    compiler = Compiler.new
+    compiler.compile(prog)
+    compiler.bytecode
+  end
+
+  def self.compile_and_run(prog, vars)
+    bytecode = compile(prog)
+    vm = VM.new(bytecode, vars)
+    vm.run
+    vm.stack_top
+  end
+end

data/examples/csv.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# A translation of the CSV example found in the pest book.
+# https://pest.rs/book/examples/csv.html
+
+# https://github.com/pest-parser/book/blob/master/LICENSE-MIT
+#
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without
+# limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software
+# is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+# ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+# IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require_relative "../lib/pestle"
+
+# NOTE: This simple example grammar requires input CSV to include a trailing
+# `\r\n` or `\n`. Without it you will get a `PestParsingError`.
+
+GRAMMAR = <<~'GRAMMAR'
+  field = { (ASCII_DIGIT | "." | "-")+ }
+  record = { field ~ ("," ~ field)* }
+  file = { SOI ~ (record ~ ("\r\n" | "\n"))* ~ EOI }
+GRAMMAR
+
+EXAMPLE_DATA = <<~CSV
+  65279,1179403647,1463895090
+  3.1415927,2.7182817,1.618034
+  -40,-273.15
+  13,42
+  65537
+CSV
+
+# The name of the rule to start parsing from. Can be a string or a symbol.
+START_RULE = :file
+
+parser = Pestle::Parser.from_grammar(GRAMMAR)
+token_pairs = parser.parse(START_RULE, EXAMPLE_DATA).first
+
+field_sum = 0.0
+record_count = 0
+
+token_pairs.each do |pair|
+  case pair
+  in :record, fields
+    record_count += 1
+    fields.each { |field| field_sum += field.text.to_f }
+  in :EOI, _
+    break
+  else
+    raise "unexpected rule #{pair.name}"
+  end
+end
+
+puts "Sum of fields: #{field_sum}"
+puts "Number of records: #{record_count}"

data/examples/ini.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# A translation of the ini example found in the pest book.
+# https://pest.rs/book/examples/ini.html
+
+# https://github.com/pest-parser/book/blob/master/LICENSE-MIT
+#
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without
+# limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software
+# is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+# ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+# TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+# IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+require "json"
+require_relative "../lib/pestle"
+
+GRAMMAR = <<~GRAMMAR
+  file = {
+      SOI ~
+      ((section | property)? ~ NEWLINE)* ~
+      EOI
+  }
+
+  char = { ASCII_ALPHANUMERIC | "." | "_" | "/" }
+  name = @{ char+ }
+  value = @{ char* }
+  section = { "[" ~ name ~ "]" }
+  property = { name ~ "=" ~ value }
+
+  WHITESPACE = _{ " " }
+GRAMMAR
+
+EXAMPLE_INI = <<~INI
+  username = noha
+  password = plain_text
+  salt = NaCl
+
+  [server_1]
+  interface=eth0
+  ip=127.0.0.1
+  document_root=/var/www/example.org
+
+  [empty_section]
+
+  [second_server]
+  document_root=/var/www/example.com
+  ip=
+  interface=eth1
+INI
+
+START_RULE = :file
+
+parser = Pestle::Parser.from_grammar(GRAMMAR)
+pairs = parser.parse(START_RULE, EXAMPLE_INI).first
+
+current_section_name = ""
+properties = Hash.new { |hash, key| hash[key] = {} }
+
+pairs.each do |pair|
+  case pair
+  in :section, [name]
+    current_section_name = name.text
+  in :property, [name, value]
+    properties[current_section_name][name.text] = value.text
+  in :EOI, _
+    break
+  else
+    raise "unexpected rule #{pair.name.inspect}"
+  end
+end
+
+puts JSON.pretty_generate(properties)