arpaka 0.1.0

data/lib/arpaka.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative "lrama/ruby"
+
+module Arpaka
+  class Error < Lrama::Ruby::Error; end
+
+  def self.generate(ruby_source:, class_name: "RubyParser")
+    Generator.new.generate(ruby_source: ruby_source, class_name: class_name)
+  end
+
+  def self.compile(ruby_source:, class_name: "RubyParser")
+    source = generate(ruby_source: ruby_source, class_name: class_name)
+    box = ::Ruby::Box.new
+    box.eval(source)
+    box.const_get(class_name, false)
+  end
+end
+
+require_relative "arpaka/generator"

data/lib/lrama/ruby/action_code.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "action_scanner"
+
+module Lrama
+  module Ruby
+    # Loaded only in the generator's box. Ruby strings, comments, regular
+    # expressions and instance variables must not become grammar references.
+    module ActionReferences
+      def ruby_short_reference?(offset)
+        references
+        @ruby_short_references.include?(offset)
+      end
+
+      private
+
+      def _references
+        result = ActionScanner.new(s_value, filename: location.grammar_file.path,
+          line: location.first_line, column: location.first_column).scan
+        @ruby_short_references = result.references.select(&:short_interpolation).map(&:first_column)
+        result.references.map do |reference|
+          attributes = if reference.number
+            { number: reference.number, index: reference.number }
+          else
+            { name: reference.name }
+          end
+          Grammar::Reference.new(type: :dollar, **attributes,
+            first_column: reference.first_column, last_column: reference.last_column)
+        end
+      rescue ActionScanner::Error => error
+        raise Backend::Error, error.message
+      end
+    end
+
+    module ActionLexer
+      def lex_c_code
+        return super unless @end_symbol == "}"
+
+        reset_first_position
+        result = ActionScanner.new(@scanner.rest, filename: @grammar_file.path,
+          line: line, column: column).scan(action: true)
+        code = @scanner.string.byteslice(@scanner.pos, result.end_offset)
+        @scanner.pos += code.bytesize
+        @line += code.count("\n")
+        @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n")
+        [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)]
+      rescue ActionScanner::Error => error
+        raise Backend::Error, error.message
+      end
+    end
+  end
+end
+
+Lrama::Lexer::Token::UserCode.prepend(Lrama::Ruby::ActionReferences)
+Lrama::Lexer.prepend(Lrama::Ruby::ActionLexer)

data/lib/lrama/ruby/action_scanner.rb

@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module Lrama
+  module Ruby
+    # A Ruby source lexer used for action boundary detection and references.
+    # All positions are bytes in the original source; literals are never decoded.
+    class ActionScanner
+      Reference = Data.define(:name, :number, :first_column, :last_column, :short_interpolation)
+      Result = Data.define(:end_offset, :references)
+      Heredoc = Data.define(:delimiter, :indent, :interpolate, :offset)
+      class Error < StandardError; end
+
+      IDENTIFIER = /[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n
+      NUMBER = /(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|[0-9][0-9_]*(?:\.[0-9_]+)?(?:[eE][+-]?[0-9_]+)?)[ri]*/
+      BEGIN_WORDS = %w[if unless while until when return yield break next rescue else elsif then do begin and or not in case for].freeze
+      END_WORDS = %w[nil true false self end __FILE__ __LINE__ __ENCODING__].freeze
+      PAIRS = { "(" => ")", "[" => "]", "{" => "}" }.freeze
+      LITERAL_PAIRS = PAIRS.merge("<" => ">").freeze
+      # Bound recursive interpolation rather than allowing SystemStackError.
+      MAX_INTERPOLATION_DEPTH = 128
+
+      def initialize(source, filename: "(grammar)", line: 1, column: 0)
+        @source = source.b
+        @scanner = StringScanner.new(@source)
+        @filename, @line, @column = filename, line, column
+        @references = []
+        @interpolation_depth = 0
+        @strict_references = false
+      end
+
+      def scan(action: false)
+        @strict_references = action
+        code(action ? "}" : nil)
+        Result.new(@scanner.pos, @references.sort_by(&:first_column).freeze)
+      end
+
+      private
+
+      def fail_at(message, offset = @scanner.pos)
+        prefix = @source.byteslice(0, offset)
+        lines = prefix.count("\n")
+        column = lines.zero? ? @column + offset : offset - prefix.rindex("\n") - 1
+        raise Error, "#{@filename}:#{@line + lines}:#{column}: #{message}"
+      end
+
+      def code(terminator)
+        brackets = []
+        heredocs = []
+        state = :begin
+        spaced = false
+        until @scanner.eos?
+          offset = @scanner.pos
+          char = @scanner.peek(1)
+          if @scanner.scan(/[ \t\r\f]+/)
+            spaced = true
+            next
+          elsif @scanner.scan(/\\\r?\n/)
+            spaced = true
+            next
+          elsif @scanner.scan(/\n/)
+            heredocs.each { |item| heredoc_body(item) }
+            heredocs.clear
+            state = :begin unless [:begin, :method].include?(state)
+            spaced = true
+            next
+          elsif char == "#"
+            @scanner.scan(/[^\n]*/)
+            next
+          elsif (offset.zero? || @source.getbyte(offset - 1) == 10) && @scanner.check(/=begin(?=\s|\z)/)
+            @scanner.scan(/[^\n]*(?:\n|\z)/)
+            found = false
+            until @scanner.eos?
+              line = @scanner.scan(/[^\n]*(?:\n|\z)/)
+              if /\A=end(?:\s|\z)/.match?(line)
+                found = true
+                break
+              end
+            end
+            fail_at("Unterminated block comment", offset) unless found
+            next
+          elsif terminator.nil? && (offset.zero? || @source.getbyte(offset - 1) == 10) &&
+            @scanner.check(/__END__(?:\s|\z)/)
+            @scanner.terminate
+            return
+          elsif char == terminator && brackets.empty?
+            fail_at("Close the action/interpolation after the heredoc body", offset) unless heredocs.empty?
+            return
+          elsif PAIRS.key?(char)
+            brackets << [PAIRS.fetch(char), offset]
+            @scanner.getch
+            state = :begin
+          elsif [")", "]", "}"].include?(char)
+            fail_at("Mismatched closing #{char}", offset) unless brackets.last&.first == char
+            brackets.pop
+            @scanner.getch
+            state = :end
+          elsif ["'", '"', "`"].include?(char)
+            @scanner.getch
+            literal(char, interpolate: char != "'", offset: offset)
+            state = :end
+          elsif char == "$"
+            reference
+            state = :end
+          elsif @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n)
+            state = :end
+          elsif @scanner.check(NUMBER)
+            @scanner.scan(NUMBER)
+            state = :end
+          elsif @scanner.check(IDENTIFIER)
+            word = @scanner.scan(IDENTIFIER)
+            fail_at("__END__ is not supported in Ruby actions", offset) if word == "__END__"
+            # Method names after dot/:: are not keywords.
+            state = if state == :method
+              :bare
+            elsif BEGIN_WORDS.include?(word)
+              :begin
+            elsif END_WORDS.include?(word)
+              :end
+            else
+              :bare
+            end
+          elsif char == "/" || char == "%" || @scanner.check(/<</)
+            if char == "%" && @scanner.check(/%[qQwWiIrsx]?[a-zA-Z0-9_]*[^a-zA-Z0-9_\\\s]/n) &&
+              (@scanner.peek(2).byteslice(1, 1).match?(/[qQwWiIrsx]/n) ||
+                !@scanner.peek(2).byteslice(1, 1).match?(/[a-zA-Z0-9_]/n))
+              percent_literal(offset)
+              state = :end
+            elsif (state == :begin || (state == :bare && spaced)) &&
+              @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)(?=\s|[;,)\]}]|\z)/n)
+              heredocs << heredoc_start(offset)
+              state = :end
+            elsif state == :method
+              @scanner.scan(/(?:<<|\/|%)/)
+              state = :bare
+            elsif state == :begin
+              if char == "/"
+                @scanner.getch
+                literal("/", interpolate: true, regexp: true, offset: offset)
+                @scanner.scan(/[a-z]*/)
+              elsif char == "%"
+                percent_literal(offset)
+              elsif @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)(?=\s|[;,)\]}]|\z)/n)
+                heredocs << heredoc_start(offset)
+              else
+                @scanner.scan(/<</)
+              end
+              state = :end
+            else
+              @scanner.scan(/(?:<<|\/|%)=?/)
+              state = :begin
+            end
+          elsif char == "?" && state == :begin
+            @scanner.getch
+            character(offset)
+            state = :end
+          elsif @scanner.scan(/&\.|::|\.(?!\.)/)
+            state = :method
+          elsif char == ":" && state == :begin && @scanner.check(/:\s/)
+            @scanner.getch
+            state = :begin
+          elsif char == ":" && state == :begin
+            @scanner.getch
+            if @scanner.check(/['"]/)
+              quote = @scanner.getch
+              literal(quote, interpolate: quote != "'", offset: offset)
+            elsif @scanner.scan(/#\{/)
+              @interpolation_depth += 1
+              code("}")
+              @scanner.getch
+              @interpolation_depth -= 1
+            elsif !@scanner.scan(/(?:@{1,2}|\$?)[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n) &&
+              !@scanner.scan(/(?:\[\]=?|<=>|===|==|=~|!~|!=|<=|>=|<<|>>|\*\*|[+\-~]@?|[!*\/%&|^<>`])/)
+              fail_at("Unsupported symbol literal", offset)
+            end
+            state = :end
+          elsif char == "@" && @scanner.check(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n)
+            @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n)
+            state = :end
+          elsif char == "@" && state == :begin
+            @scanner.getch
+            state = :end
+          elsif @scanner.scan(/(?:\.\.\.?|->|\*\*=|&&=?|\|\|=?|<=>|===|==|=>|!=|!~|=~|<=|>=|>>=?|\*\*|[+\-*|&^]=|[=+\-*!,;:<>?~|&^])/)
+            state = :begin
+          else
+            fail_at("Unsupported Ruby action token #{char.inspect}", offset)
+          end
+          spaced = false
+        end
+        fail_at("Unterminated heredoc", heredocs.first.offset) unless heredocs.empty?
+        fail_at("Unclosed #{brackets.last.first}", brackets.last.last) unless brackets.empty?
+        fail_at("Unterminated Ruby action/interpolation") if terminator
+      end
+
+      def reference(short: false)
+        offset = @scanner.pos
+        @global_variable_start = offset
+        value = @scanner.scan(/\$(?:\$|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/)
+        unless value
+          return scan_global_variable unless @strict_references
+          fail_at("Unsupported semantic reference", offset)
+        end
+        if value.match?(/\A\$0/) || @scanner.check(/[a-zA-Z_0-9\x80-\xff]/n)
+          return scan_global_variable unless @strict_references
+          fail_at("Unsupported semantic reference", offset)
+        end
+        number = value.match?(/\A\$[0-9]+\z/) ? value.delete_prefix("$").to_i : nil
+        @references << Reference.new(number ? nil : value.delete_prefix("$"), number,
+          offset, @scanner.pos, short)
+      end
+
+      def scan_global_variable
+        @scanner.pos = @global_variable_start || @scanner.pos
+        @scanner.scan(/\$(?:`|')/) ||
+          @scanner.scan(/\$(?:[<>]?|[!@&+~?=\/\\;,.:$-][a-zA-Z]?|[<>][^>\n]*>|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/)
+        fail_at("Unsupported global variable", @scanner.pos) if @scanner.pos == (@global_variable_start || @scanner.pos)
+      end
+
+      def interpolation
+        return false unless @scanner.peek(1) == "#"
+        if @scanner.scan(/#\{/)
+          @interpolation_depth += 1
+          fail_at("Interpolation nesting exceeds #{MAX_INTERPOLATION_DEPTH}") if @interpolation_depth > MAX_INTERPOLATION_DEPTH
+          code("}")
+          @scanner.getch
+          @interpolation_depth -= 1
+        elsif @scanner.scan(/#(?=\$)/)
+          reference(short: true)
+        elsif @scanner.scan(/#@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n)
+          # Instance/class variable interpolation is preserved verbatim.
+        else
+          return false
+        end
+        true
+      end
+
+      def literal(close, interpolate:, offset:, open: nil, regexp: false)
+        depth = 0
+        until @scanner.eos?
+          char = @scanner.peek(1)
+          if char == "\\"
+            @scanner.getch
+            @scanner.getch || fail_at("Unterminated escape", offset)
+          elsif interpolate && interpolation
+            next
+          elsif regexp && char == "["
+            regexp_class(offset)
+          elsif char == close
+            @scanner.getch
+            return if depth.zero?
+            depth -= 1
+          elsif open && char == open
+            depth += 1
+            @scanner.getch
+          else
+            @scanner.getch
+          end
+        end
+        fail_at("Unterminated literal (expected #{close})", offset)
+      end
+
+      def regexp_class(offset)
+        @scanner.getch
+        @scanner.scan(/\^/)
+        @scanner.scan(/\]/) # A leading ] is literal.
+        until @scanner.eos?
+          if @scanner.scan(/\\[\s\S]/)
+            next
+          elsif interpolation
+            next
+          elsif @scanner.scan(/\[:[^\]\n]*:\]/)
+            next # POSIX character class inside a bracket expression.
+          elsif @scanner.scan(/\]/)
+            return
+          else
+            @scanner.getch
+          end
+        end
+        fail_at("Unterminated regexp character class", offset)
+      end
+
+      def percent_literal(offset)
+        @scanner.getch
+        kind = @scanner.scan(/[qQwWiIrsx]/) || "Q"
+        delimiter = @scanner.getch
+        unless delimiter && delimiter.bytesize == 1 &&
+          !delimiter.match?(/[a-zA-Z0-9\\\r\n]/n)
+          fail_at("Unsupported percent literal delimiter", offset)
+        end
+        literal(LITERAL_PAIRS.fetch(delimiter, delimiter), open: LITERAL_PAIRS.key?(delimiter) ? delimiter : nil,
+          interpolate: !%w[q w i s].include?(kind), regexp: kind == "r", offset: offset)
+        @scanner.scan(/[a-z]*/) if kind == "r"
+      end
+
+      def heredoc_start(offset)
+        @scanner.scan(/<</)
+        indent = !!@scanner.scan(/[-~]/)
+        quote = @scanner.scan(/['"`]/)
+        delimiter = if quote
+          start = @scanner.pos
+          @scanner.getch until @scanner.eos? || [quote, "\r", "\n"].include?(@scanner.peek(1))
+          text = @source.byteslice(start, @scanner.pos - start)
+          fail_at("Unterminated heredoc delimiter", offset) unless @scanner.getch == quote
+          text
+        else
+          @scanner.scan(/[a-zA-Z_][a-zA-Z_0-9]*/)
+        end
+        fail_at("Unsupported heredoc delimiter", offset) unless delimiter && !delimiter.empty?
+        Heredoc.new(delimiter, indent, quote != "'", offset)
+      end
+
+      def heredoc_body(item)
+        until @scanner.eos?
+          if @scanner.pos.zero? || @source.getbyte(@scanner.pos - 1) == 10
+            start = @scanner.pos
+            @scanner.scan(/[ \t]*/) if item.indent
+            if @scanner.peek(item.delimiter.bytesize) == item.delimiter
+              @scanner.pos += item.delimiter.bytesize
+              return if @scanner.eos? || @scanner.scan(/\r?\n/)
+            end
+            @scanner.pos = start
+          end
+          if item.interpolate && interpolation
+            next
+          elsif item.interpolate && @scanner.scan(/\\[\s\S]/)
+            next
+          else
+            @scanner.getch
+          end
+        end
+        fail_at("Unterminated heredoc #{item.delimiter}", item.offset)
+      end
+
+      def character(offset)
+        fail_at("Unterminated character literal", offset) if @scanner.eos? || @scanner.check(/\s/)
+        if @scanner.scan(/\\/)
+          if @scanner.scan(/u\{/)
+            fail_at("Unterminated character escape", offset) unless @scanner.scan(/[0-9a-fA-F \t]+\}/)
+          elsif @scanner.scan(/(?:[CM]-|c)/)
+            character(offset)
+          else
+            @scanner.getch || fail_at("Unterminated character escape", offset)
+          end
+        else
+          # Consume exactly one UTF-8 codepoint (or one ASCII byte).
+          @scanner.scan(/(?:[\xc2-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xf4][\x80-\xbf]{3}|[\x00-\x7f])/n) ||
+            fail_at("Unsupported character encoding", offset)
+        end
+      end
+    end
+  end
+end

data/lib/lrama/ruby/backend.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "lrama"
+require "erb"
+
+module Lrama
+  module Ruby
+    class Backend
+      class Error < StandardError; end
+
+      def generate(source, filename:, class_name:, mode:, allow_error_rules: false)
+        recognizer = mode == :recognizer
+        grammar = Lrama::Parser.new(source, filename).parse
+        unless grammar.no_stdlib
+          path = Lrama::Command::STDLIB_FILE_PATH
+          stdlib = Lrama::Parser.new(File.read(path), path).parse
+          grammar.prepend_parameterized_rules(stdlib.parameterized_rules)
+        end
+        grammar.prepare
+        grammar.validate!
+        validate_features!(grammar, allow_error_rules: allow_error_rules) unless recognizer
+
+        states = Lrama::States.new(grammar, Lrama::Tracer.new($stderr))
+        states.compute
+        states.compute_ielr if grammar.ielr_defined?
+        if states.rr_conflicts_count.positive? || states.sr_conflicts_count != (grammar.expect || 0)
+          raise Error, "Unresolved conflicts: #{states.sr_conflicts_count} shift/reduce, #{states.rr_conflicts_count} reduce/reduce"
+        end
+        context = Lrama::Context.new(states)
+        tokens = token_names(grammar)
+        actions = grammar.rules.filter_map do |rule|
+          next if recognizer || !rule.token_code
+          # Preserve whitespace inside multiline strings and heredocs.
+          "    when #{rule.id + 1} # #{rule.as_comment.gsub(/[\r\n]/, ' ')}\n#{translate_action(rule)}\n"
+        end.join
+        template = File.read(File.expand_path("parser.rb.erb", __dir__))
+        result = ::ERB.new(template, trim_mode: "-").result_with_hash(
+          class_name: class_name, context: context, tokens: tokens, actions: actions,
+          recognizer: recognizer
+        )
+        result
+      end
+
+      private
+
+      def translate_action(rule)
+        code = rule.token_code.s_value.dup
+        position = rule.position_in_original_rule_rhs || rule.rhs.length
+        rule.token_code.references.reverse_each do |reference|
+          replacement = if reference.name == "$"
+            "yyval"
+          else
+            "values[#{reference.index - position - 1}]"
+          end
+          if rule.token_code.ruby_short_reference?(reference.first_column)
+            replacement = "{#{replacement}}"
+          end
+          # Lrama and action scanner locations are byte offsets, including UTF-8.
+          code = code.byteslice(0, reference.first_column) + replacement +
+            code.byteslice(reference.last_column..)
+        end
+        code
+      end
+
+      def token_names(grammar)
+        grammar.terms.each_with_object({}) do |term, names|
+          next if term.error_symbol? || term.undef_symbol?
+          names[term.id.s_value] = term.token_id
+          if term.id.is_a?(Lrama::Lexer::Token::Char)
+            names[term.token_id.chr(Encoding::UTF_8)] = term.token_id
+          end
+          if term.alias_name&.start_with?('"')
+            names[term.alias_name[1...-1]] = term.token_id
+          end
+        end
+      end
+
+      def validate_features!(grammar, allow_error_rules:)
+        unsupported = []
+        unsupported << "%locations" if grammar.locations
+        unsupported << "%union" if grammar.union
+        unsupported << "%code" unless grammar.percent_codes.empty?
+        unsupported << "prologue" unless grammar.aux.prologue.to_s.strip.empty?
+        unsupported << "epilogue" unless grammar.aux.epilogue.to_s.strip.empty?
+        {
+          "%parse-param" => grammar.parse_param, "%lex-param" => grammar.lex_param,
+          "%initial-action" => grammar.initial_action, "%after-shift" => grammar.after_shift,
+          "%before-reduce" => grammar.before_reduce, "%after-reduce" => grammar.after_reduce,
+          "%after-shift-error-token" => grammar.after_shift_error_token,
+          "%after-pop-stack" => grammar.after_pop_stack
+        }.each { |name, value| unsupported << name if value }
+        unsupported << "%printer" unless grammar.printers.empty?
+        unsupported << "%error-token" unless grammar.error_tokens.empty?
+        unsupported << "%destructor" if grammar.symbols.any?(&:destructor)
+        unsupported << "typed symbols" if grammar.symbols.any?(&:tag)
+        if !allow_error_rules && grammar.rules.any? { |rule| rule.rhs.include?(grammar.error_symbol) }
+          unsupported << "error recovery rules"
+        end
+        raise Error, "Not supported by the Ruby backend yet: #{unsupported.join(', ')}" unless unsupported.empty?
+      end
+    end
+  end
+end

data/lib/lrama/ruby/generator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "lrama"
+
+module Lrama
+  module Ruby
+    class Generator
+      def initialize
+        unless ::Ruby::Box.enabled?
+          raise Error, "Ruby Box is required. Start Ruby with RUBY_BOX=1."
+        end
+      end
+
+      def generate(source, filename: "(grammar)", class_name: "Parser", mode: :parser, allow_error_rules: false)
+        unless /\A[A-Z][a-zA-Z0-9_]*\z/.match?(class_name)
+          raise Error, "class_name must be a single Ruby constant name"
+        end
+
+        unless [:parser, :recognizer].include?(mode)
+          raise Error, "mode must be :parser or :recognizer"
+        end
+
+        box = ::Ruby::Box.new
+        # A new box does not inherit Bundler's activated load paths.
+        box.load_path.replace($LOAD_PATH)
+        box.require(File.expand_path("backend.rb", __dir__))
+        box.require(File.expand_path("action_code.rb", __dir__)) if mode == :parser
+        begin
+          box::Lrama::Ruby::Backend.new.generate(source,
+            filename: filename, class_name: class_name, mode: mode,
+            allow_error_rules: allow_error_rules)
+        rescue box::Lrama::Ruby::Backend::Error => e
+          raise Error, e.message
+        end
+      end
+    end
+  end
+end