arpaka 0.1.0

data/lib/arpaka/cli.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "tempfile"
+require_relative "../arpaka"
+
+module Arpaka
+  class CLI
+    def self.run(argv, out: $stdout, err: $stderr)
+      options = { class_name: "RubyParser" }
+      force = false
+      help = false
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: arpaka generate --ruby-source DIR --output FILE [--class-name NAME] [--force]"
+        opts.on("--ruby-source DIR", "Ruby source tree (provided by the user)") { |v| options[:ruby_source] = v }
+        opts.on("--output FILE", "Write a standalone Ruby frontend") { |v| options[:output] = v }
+        opts.on("--class-name NAME", "Generated class name (default: RubyParser)") { |v| options[:class_name] = v }
+        opts.on("--force", "Replace an existing output file") { force = true }
+        opts.on("-h", "--help", "Show usage") { help = true }
+      end
+      args = argv.dup
+      command = args.shift
+      if ["--help", "-h"].include?(command)
+        out.puts(parser)
+        return 0
+      end
+      raise OptionParser::InvalidArgument, "expected generate" unless command == "generate"
+      parser.parse!(args)
+      if help
+        out.puts(parser)
+        return 0
+      end
+      raise OptionParser::InvalidArgument, args.join(" ") unless args.empty?
+      %i[ruby_source output].each do |key|
+        raise OptionParser::MissingArgument, "--#{key.to_s.tr('_', '-')}" unless options[key]
+      end
+      output = File.expand_path(options.delete(:output))
+      if !force && (File.exist?(output) || File.symlink?(output))
+        raise Error, "Output already exists: #{output}; use --force to replace it"
+      end
+      source = Arpaka.generate(**options)
+      Tempfile.create([".arpaka-", ".rb"], File.dirname(output)) do |file|
+        file.write(source)
+        file.flush
+        file.chmod(0o644)
+        if force
+          File.rename(file.path, output)
+        else
+          # Publish only the complete file, without overwriting a concurrent writer.
+          File.link(file.path, output)
+        end
+      end
+      out.puts("Generated #{output}")
+      0
+    rescue OptionParser::ParseError, Lrama::Ruby::Error, SystemCallError => error
+      err.puts("arpaka: #{error.message}")
+      1
+    end
+  end
+end

data/lib/arpaka/generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "erb"
+require "json"
+require_relative "ruby_grammar"
+
+module Arpaka
+  class Generator
+    def generate(ruby_source:, class_name:)
+      unless /\A[A-Z][a-zA-Z0-9_]*\z/.match?(class_name)
+        raise Error, "class_name must be a single Ruby constant name"
+      end
+      unless ::Ruby::Box.enabled?
+        raise Error, "Ruby Box is required for generation. Start Ruby with RUBY_BOX=1."
+      end
+      artifacts = RubyGrammar.artifacts(ruby_source: ruby_source)
+      parser = Lrama::Ruby.generate(artifacts.fetch("parse.y"),
+        filename: File.join(ruby_source, "parse.y"), class_name: "Parser", allow_error_rules: true)
+      parser = parser.delete_prefix("# frozen_string_literal: true\n")
+      rules = JSON.parse(artifacts.fetch("rules.json"))
+      runtime = %w[frontend ast builder lexer].map do |name|
+        template = File.read(File.join(__dir__, "runtime", "#{name}.rb.erb"))
+        name == "builder" ? ERB.new(template).result_with_hash(rules: rules) : template
+      end.join("\n")
+      notices = [File.read(File.expand_path("../../LICENSE.txt", __dir__))]
+      %w[COPYING BSDL].each { |name| notices << File.read(File.join(ruby_source, name)) }
+      source = "# frozen_string_literal: true\n" \
+        "# Generated by Arpaka #{Lrama::Ruby::VERSION}. Do not edit.\n" \
+        "# Ruby source profile: #{RubyGrammar::REVISION}\n" \
+        "# parse.y SHA256: #{Digest::SHA256.file(File.join(ruby_source, 'parse.y')).hexdigest}\n" \
+        "#{notices.join("\n").lines.map { |line| "# #{line}" }.join}\n" \
+        "class #{class_name}\n#{parser}\n#{runtime}\nend\n"
+      # Compile in an isolated box to validate syntax without executing actions.
+      box = ::Ruby::Box.new
+      box.eval(source)
+      source
+    rescue SyntaxError => error
+      raise Error, "Invalid generated frontend: #{error.message}"
+    rescue SystemCallError => error
+      raise Error, "Cannot read Ruby frontend inputs: #{error.message}"
+    end
+  end
+end

data/lib/arpaka/ruby_grammar.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "lrama"
+require "json"
+require "digest"
+require "open3"
+require "rbconfig"
+require_relative "actions"
+
+module Arpaka::RubyGrammar
+  REVISION = "37d60dd3241d5fdb10ca43f36077f5d556ae1b8d"
+  EXTENDED_RULES = {
+    ["value_expr_command", ["command"]] => "tLBRACE_ARG"
+  }.freeze
+
+  def self.parse(source, filename)
+    grammar = Lrama::Parser.new(source, filename).parse
+    unless grammar.no_stdlib
+      stdlib = Lrama::Command::STDLIB_FILE_PATH
+      grammar.prepend_parameterized_rules(Lrama::Parser.new(File.read(stdlib), stdlib).parse.parameterized_rules)
+    end
+    grammar.prepare
+    grammar.validate!
+    grammar
+  end
+
+  def self.load_upstream(ruby_source)
+    vendor = File.expand_path(ruby_source)
+    manifest = JSON.parse(File.read(File.join(__dir__, "source.json")))
+    raise ::Arpaka::Error, "Unexpected upstream revision" unless manifest.fetch("revision") == REVISION
+    manifest.fetch("files").each do |path, digest|
+      actual = Digest::SHA256.file(File.join(vendor, path)).hexdigest
+      next if actual == digest
+
+      raise ::Arpaka::Error, "Unsupported Ruby source change: #{path}; " \
+        "expected SHA256 #{digest}, got #{actual}. " \
+        "Update the Action mappings and source profile before regenerating."
+    end
+    source, error, result = Open3.capture3(RbConfig.ruby,
+      File.join(vendor, "tool/id2token.rb"), File.join(vendor, "parse.y"))
+    raise ::Arpaka::Error, "Ruby preprocessing failed: #{error}" unless result.success?
+    grammar = parse(source, File.join(vendor, "parse.y"))
+    inventory = grammar.rules.reject(&:initial_rule?).map do |rule|
+      { "id" => rule.id, "lhs" => rule.lhs.id.s_value,
+        "rhs" => rule.rhs.map { |symbol| symbol.id.s_value },
+        "line" => rule.token_code&.location&.first_line || rule.lineno,
+        "code" => rule.token_code&.s_value,
+        "midrule_position" => rule.position_in_original_rule_rhs,
+        "precedence" => rule.precedence_sym&.id&.s_value }
+    end
+    expected = JSON.parse(File.read(File.join(__dir__, "upstream_rules.json")))
+    unless inventory == expected
+      changed = (0...[inventory.length, expected.length].max).select { |i| inventory[i] != expected[i] }
+      details = changed.first(5).map do |i|
+        before, after = expected[i], inventory[i]
+        "rule #{before&.fetch('id') || after.fetch('id')}: " \
+          "#{before&.fetch('lhs') || '(absent)'} -> #{after&.fetch('lhs') || '(absent)'}"
+      end
+      raise ::Arpaka::Error, "Upstream rules changed (#{changed.length} entries); " \
+        "review Action mappings: #{details.join('; ')}"
+    end
+    unknown = ::Arpaka::RubyGrammarActions::ACTIONS.keys - inventory.map { |rule| rule.fetch("id") }
+    raise ::Arpaka::Error, "Unknown action IDs: #{unknown.inspect}" unless unknown.empty?
+    [grammar, inventory]
+  end
+
+  def self.symbol_names(grammar)
+    used = grammar.terms.map { |symbol| symbol.id.s_value }
+    grammar.nterms.to_h do |symbol|
+      original = symbol.id.s_value
+      candidate = if original.match?(/\A\$?@\d+\z/)
+        "midrule_#{original.delete('$@')}"
+      else
+        original.gsub("'\\n'", "newline").gsub(/[^a-zA-Z0-9_]/, "_")
+      end
+      candidate = "rule_#{candidate}" unless candidate.match?(/\A[a-zA-Z_]/)
+      candidate += "_#{symbol.number}" if used.include?(candidate)
+      raise ::Arpaka::Error, "Symbol name collision: #{candidate}" if used.include?(candidate)
+      used << candidate
+      [original, candidate]
+    end
+  end
+
+  def self.artifacts(ruby_source:)
+    grammar, inventory = load_upstream(ruby_source)
+    names = symbol_names(grammar)
+    name = ->(symbol) { symbol.term ? symbol.id.s_value : names.fetch(symbol.id.s_value) }
+    lines = ["/* Generated from ruby/ruby #{REVISION}. See Arpaka Action mappings. */",
+      "%no-stdlib", "%expect #{grammar.expect || 0}"]
+    lines << "%define lr.type ielr" if grammar.ielr_defined?
+    grammar.terms.each do |symbol|
+      next if symbol.error_symbol? || symbol.undef_symbol?
+      declaration = "%token #{name.call(symbol)} #{symbol.token_id}"
+      declaration += " #{symbol.alias_name}" if symbol.alias_name
+      lines << declaration
+    end
+    grammar.precedences.group_by(&:precedence).sort.each do |_level, group|
+      lines << "%#{group.first.type} #{group.map(&:s_value).join(' ')}"
+    end
+    lines << "%left keyword_do_block"
+    lines << "%start #{name.call(grammar.rules.first.rhs.first)}"
+    lines << "%%"
+    metadata = inventory.to_h { |rule| [rule.fetch("id"), rule] }
+    rules = grammar.rules.reject(&:initial_rule?)
+    rules.each do |rule|
+      rhs = rule.rhs.empty? ? "%empty" : rule.rhs.map(&name).join(" ")
+      extended_precedence = EXTENDED_RULES[[rule.lhs.id.s_value, rule.rhs.map { |symbol| symbol.id.s_value }]]
+      precedence = if rule.precedence_sym
+        " %prec #{name.call(rule.precedence_sym)}"
+      elsif extended_precedence
+        " %prec #{extended_precedence}"
+      else
+        ""
+      end
+      expression = ::Arpaka::RubyGrammarActions::ACTIONS[rule.id]&.last || "@builder.unsupported(#{rule.id})"
+      lines << "/* upstream parse.y:#{metadata.fetch(rule.id).fetch('line')}: #{rule.as_comment} */"
+      lines << "#{name.call(rule.lhs)}: #{rhs}#{precedence} { $$ = #{expression} };"
+    end
+    lines << "/* arpaka extension: block call as a parenthesized argument */"
+    lines << "call_args: block_call { $$ = [$1].freeze };"
+    lines << "call_args: args ',' block_call %prec tLOWEST { $$ = ($1 + [$3]).freeze };"
+    source = lines.join("\n") + "\n"
+    verify_structure(grammar, source, names)
+    compact = rules.map do |rule|
+      { id: rule.id, rule: rule.as_comment,
+        line: metadata.fetch(rule.id).fetch("line"),
+        status: ::Arpaka::RubyGrammarActions::ACTIONS[rule.id]&.first || "unsupported" }
+    end
+    { "parse.y" => source, "rules.json" => JSON.pretty_generate(compact) + "\n" }
+  end
+
+  def self.verify_structure(original, source, names)
+    # C-lexer inspection only: @builder is a Ruby ivar, not a yacc location.
+    port = parse(source.gsub("@builder.", "builder_"), "generated Ruby grammar")
+    signature = lambda do |grammar, rename|
+      grammar.rules.reject(&:initial_rule?).map do |rule|
+        name = ->(symbol) { rename.fetch(symbol.id.s_value, symbol.id.s_value) }
+        [name.call(rule.lhs), rule.rhs.map(&name), rule.precedence_sym && name.call(rule.precedence_sym)]
+      end.sort_by(&:inspect)
+    end
+    expected_rules = signature.call(original, names).map do |rule|
+      if rule[0] == names.fetch("value_expr_command") && rule[1] == [names.fetch("command")]
+        rule[0, 2] + ["tLBRACE_ARG"]
+      else
+        rule
+      end
+    end
+    expected_rules << ["call_args", ["block_call"], nil]
+    expected_rules << ["call_args", ["args", "','", "block_call"], "tLOWEST"]
+    actual_rules = signature.call(port, {})
+    unless expected_rules.sort_by(&:inspect) == actual_rules.sort_by(&:inspect)
+      raise ::Arpaka::Error, "Production structure changed"
+    end
+    terms = lambda do |grammar|
+      grammar.terms.map { |symbol| [symbol.id.s_value, symbol.token_id, symbol.precedence&.type, symbol.precedence&.precedence] }.sort_by(&:inspect)
+    end
+    expected_terms = terms.call(original).map do |term|
+      if term[0] == "keyword_do_block"
+        [term[0], term[1], :left, original.precedences.map(&:precedence).max + 1]
+      else
+        term
+      end
+    end
+    actual_terms = terms.call(port)
+    unless expected_terms.sort_by(&:inspect) == actual_terms
+      raise ::Arpaka::Error, "Tokens or precedence changed: expected=#{expected_terms.sort_by(&:inspect).inspect} actual=#{actual_terms.inspect}"
+    end
+  end
+
+end

data/lib/arpaka/runtime/ast.rb.erb

@@ -0,0 +1,35 @@
+module AST
+  Program = Data.define(:statements)
+  Literal = Data.define(:value)
+  Binary = Data.define(:operator, :left, :right)
+  Unary = Data.define(:operator, :operand)
+  If = Data.define(:condition, :then_body, :else_body)
+  LocalRead = Data.define(:name)
+  LocalWrite = Data.define(:name, :value)
+  BareCall = Data.define(:name)
+  ArrayLiteral = Data.define(:elements)
+  HashLiteral = Data.define(:pairs)
+  Pair = Data.define(:key, :value)
+  RangeLiteral = Data.define(:operator, :begin_value, :end_value)
+  Ternary = Data.define(:condition, :then_value, :else_value)
+  Call = Data.define(:name, :arguments)
+  ReceiverCall = Data.define(:receiver, :operator, :name, :arguments)
+  BlockCall = Data.define(:call, :body)
+  StringLiteral = Data.define(:value)
+  InterpolatedString = Data.define(:parts)
+  Index = Data.define(:receiver, :arguments)
+  Loop = Data.define(:kind, :condition, :body)
+  Control = Data.define(:kind)
+  Variable = Data.define(:kind, :name)
+  Rescue = Data.define(:expression, :fallback)
+  RegexpLiteral = Data.define(:value)
+  XStringLiteral = Data.define(:value)
+  For = Data.define(:variable, :enumerable, :body)
+  Def = Data.define(:name, :arguments, :body)
+  ClassDef = Data.define(:name, :superclass, :body)
+  ModuleDef = Data.define(:name, :body)
+  When = Data.define(:patterns, :body)
+  Case = Data.define(:expression, :whens, :else_body)
+  Lambda = Data.define(:arguments, :body)
+  Sequence = Data.define(:statements)
+end

data/lib/arpaka/runtime/builder.rb.erb

@@ -0,0 +1,247 @@
+class Builder
+  RULES = <%= rules.inspect %>.to_h { |rule| [rule.fetch("id"), rule.freeze] }.freeze
+  private_constant :RULES
+
+  def initialize
+    @locals = {}
+  end
+
+  def program(statements)
+    AST::Program.new(statements.freeze)
+  end
+
+  def integer(value)
+    raise ArgumentError, "tINTEGER requires an Integer value" unless value.is_a?(Integer)
+    AST::Literal.new(value)
+  end
+
+  def float(value)
+    raise ArgumentError, "tFLOAT requires a Float value" unless value.is_a?(Float)
+    AST::Literal.new(value)
+  end
+
+  def literal(value)
+    AST::Literal.new(value)
+  end
+
+  def reserved_word(value)
+    value.to_sym
+  end
+
+  def binary(operator, left, right)
+    AST::Binary.new(operator, left, right)
+  end
+
+  def operator(value)
+    value.to_sym
+  end
+
+  def range(operator, begin_value, end_value)
+    AST::RangeLiteral.new(operator, begin_value, end_value)
+  end
+
+  def ternary(condition, then_value, else_value)
+    AST::Ternary.new(condition, then_value, else_value)
+  end
+
+  def call(name, arguments)
+    AST::Call.new(name, arguments.freeze)
+  end
+
+  def receiver_call(receiver, operator, name, arguments)
+    AST::ReceiverCall.new(receiver, operator, name, (arguments || []).freeze)
+  end
+
+  def block_call(call, body)
+    AST::BlockCall.new(call, body_nodes(body))
+  end
+
+  def append_arguments(arguments, argument)
+    list = arguments.is_a?(Array) ? arguments : [arguments]
+    (list + [argument]).freeze
+  end
+
+  def string(value)
+    if value.is_a?(Array)
+      return AST::InterpolatedString.new(value.freeze) if value.any? { |part| !part.is_a?(String) }
+      value = value.join
+    end
+    AST::StringLiteral.new(value.to_s)
+  end
+
+  def regexp(value)
+    AST::RegexpLiteral.new(value.is_a?(Array) ? value.join : value.to_s)
+  end
+
+  def xstring(value)
+    AST::XStringLiteral.new(value.is_a?(Array) ? value.join : value.to_s)
+  end
+
+  def for_node(variable, enumerable, body)
+    AST::For.new(variable, enumerable, body)
+  end
+
+  def def_node(name, arguments, body)
+    AST::Def.new(name.to_sym, (arguments || []).freeze, body || [].freeze)
+  end
+
+  def class_node(name, superclass, body)
+    AST::ClassDef.new(name.to_sym, superclass, body_nodes(body))
+  end
+
+  def module_node(name, body)
+    AST::ModuleDef.new(name.to_sym, body_nodes(body))
+  end
+
+  def body_nodes(body)
+    value = body
+    while value.is_a?(Array) && value.length == 1 && value.first.is_a?(Array)
+      value = value.first
+    end
+    value || [].freeze
+  end
+
+  def symbol(value)
+    value = value.map { |part| part.is_a?(String) ? part : part.to_s }.join if value.is_a?(Array)
+    value = value.name if value.is_a?(AST::Variable)
+    AST::Literal.new(value.to_sym)
+  end
+
+  def join_strings(parts)
+    parts
+  end
+
+  def interpolation(value)
+    value.is_a?(Array) && value.length == 1 ? value.first : value
+  end
+
+  def concat_strings(left, right)
+    left_parts = left.is_a?(AST::InterpolatedString) ? left.parts : [left.value]
+    right_parts = right.is_a?(AST::InterpolatedString) ? right.parts : [right.value]
+    parts = left_parts + right_parts
+    parts.all? { |part| part.is_a?(String) } ? AST::StringLiteral.new(parts.join) : AST::InterpolatedString.new(parts.freeze)
+  end
+
+  def index(receiver, arguments)
+    AST::Index.new(receiver, arguments.freeze)
+  end
+
+  def loop(kind, condition, body)
+    AST::Loop.new(kind, condition, body)
+  end
+
+  def control(kind)
+    AST::Control.new(kind)
+  end
+
+  def variable(kind, name)
+    AST::Variable.new(kind, name.is_a?(String) ? name.to_sym : name)
+  end
+
+  def rescue_modifier(expression, fallback)
+    AST::Rescue.new(expression, fallback)
+  end
+
+  def when_node(patterns, body)
+    AST::When.new(patterns.freeze, body)
+  end
+
+  def case_node(expression, whens, else_body)
+    AST::Case.new(expression, whens.freeze, else_body)
+  end
+
+  def case_clauses(value)
+    value.is_a?(Array) ? value : [value]
+  end
+
+  def case_parts(value)
+    values = case_clauses(value)
+    if values.all? { |item| item.is_a?(AST::When) }
+      [values, nil]
+    else
+      [values.grep(AST::When), values.reject { |item| item.is_a?(AST::When) }]
+    end
+  end
+
+  def case_chain(when_node, rest)
+    [when_node] + (rest.nil? ? [] : case_clauses(rest))
+  end
+
+  def lambda_node(arguments, body)
+    AST::Lambda.new((arguments || []).freeze, body_nodes(body))
+  end
+
+  def unary(operator, operand)
+    AST::Unary.new(operator, operand)
+  end
+
+  def if_node(condition, then_body, else_body)
+    AST::If.new(condition, then_body, else_body)
+  end
+
+  def unless_node(condition, then_body, else_body)
+    AST::If.new(AST::Unary.new(:!, condition), then_body, else_body)
+  end
+
+  def elsif_node(condition, then_body, else_body)
+    AST::If.new(condition, then_body, else_body)
+  end
+
+  def parentheses(statements, rule_id)
+    return statements.first if statements.length == 1
+    AST::Sequence.new(statements.freeze)
+  end
+
+  def identifier(value)
+    unless value.is_a?(Symbol) || value.is_a?(String)
+      raise ArgumentError, "tIDENTIFIER requires a Symbol or String value"
+    end
+    value.to_sym
+  end
+
+  def declare_local(name)
+    @locals[name] = true
+    name
+  end
+
+  def read_local(name)
+    return name if name.is_a?(AST::Variable)
+    (@locals.key?(name) ? AST::LocalRead : AST::BareCall).new(name)
+  end
+
+  def assign(name, value)
+    AST::LocalWrite.new(name, value)
+  end
+
+  def array(elements)
+    AST::ArrayLiteral.new(elements.freeze)
+  end
+
+  def word_array(elements, symbols: false)
+    elements ||= []
+    elements = elements.elements if elements.is_a?(AST::ArrayLiteral)
+    values = elements.map do |value|
+      value = value.value if value.is_a?(AST::StringLiteral)
+      symbols ? value.to_sym : AST::StringLiteral.new(value.to_s)
+    end
+    AST::ArrayLiteral.new(values.freeze)
+  end
+
+  def hash(pairs)
+    AST::HashLiteral.new(pairs.freeze)
+  end
+
+  def pair(key, value)
+    AST::Pair.new(key, value)
+  end
+
+  def append(list, value)
+    value.nil? ? list : (list + [value]).freeze
+  end
+
+  def unsupported(id)
+    rule = RULES.fetch(id)
+    raise UnsupportedSyntax.new(rule.fetch("rule"), rule.fetch("line"))
+  end
+end
+private_constant :Builder

data/lib/arpaka/runtime/frontend.rb.erb

@@ -0,0 +1,37 @@
+class Error < StandardError; end
+
+class ParseError < Error
+  attr_reader :token, :state
+
+  def initialize(error)
+    @token, @state = error.token, error.state
+    super(error.message)
+  end
+end
+
+class UnsupportedSyntax < Error
+  attr_reader :rule, :line
+
+  def initialize(rule, line)
+    @rule, @line = rule, line
+    super("Unsupported syntax: #{rule} (upstream parse.y:#{line})")
+  end
+end
+
+class LexerError < Error; end
+
+def self.parse(source, filename: "(ruby)")
+  lexer = Lexer.new(source, filename: filename)
+  parse_tokens(lexer.each, lexical_context: lexer.context)
+end
+
+def self.parse_tokens(tokens, lexical_context: nil)
+  parser = Parser.new
+  parser.instance_variable_set(:@builder, Builder.new)
+  parser.lexical_context = lexical_context
+  parser.parse(tokens)
+rescue Parser::ParseError => error
+  raise ParseError.new(error)
+end
+private_class_method :parse_tokens
+private_constant :Parser