arpaka 0.1.1 → 1.0.0

data/lib/arpaka/ruby_grammar.rb

@@ -3,16 +3,14 @@
 require "lrama"
 require "json"
 require "digest"
-require "open3"
-require "rbconfig"
 require_relative "actions"
+require_relative "profile/ruby"
 
 module Arpaka::RubyGrammar
   REVISION = "37d60dd3241d5fdb10ca43f36077f5d556ae1b8d"
   EXTENDED_RULES = {
     ["value_expr_command", ["command"]] => "tLBRACE_ARG"
   }.freeze
-
   def self.parse(source, filename)
     grammar = Lrama::Parser.new(source, filename).parse
     unless grammar.no_stdlib
@@ -35,10 +33,20 @@ module Arpaka::RubyGrammar
       warn "Arpaka: Ruby source differs from the recorded profile: #{path}; " \
         "expected SHA256 #{digest}, got #{actual}."
     end
-    source, error, result = Open3.capture3(RbConfig.ruby,
-      File.join(vendor, "tool/id2token.rb"), File.join(vendor, "parse.y"))
-    raise ::Arpaka::Error, "Ruby preprocessing failed: #{error}" unless result.success?
+    source = preprocess(File.read(File.join(vendor, "parse.y")),
+      File.join(vendor, "defs/id.def"))
     grammar = parse(source, File.join(vendor, "parse.y"))
+    [grammar, inventory(grammar)]
+  end
+
+  def self.load_parse_y(parse_y)
+    path = File.expand_path(parse_y)
+    source = preprocess_names(File.read(path))
+    grammar = parse(source, path)
+    [grammar, inventory(grammar)]
+  end
+
+  def self.inventory(grammar)
     inventory = grammar.rules.reject(&:initial_rule?).map do |rule|
       { "id" => rule.id, "lhs" => rule.lhs.id.s_value,
         "rhs" => rule.rhs.map { |symbol| symbol.id.s_value },
@@ -47,9 +55,34 @@ module Arpaka::RubyGrammar
         "midrule_position" => rule.position_in_original_rule_rhs,
         "precedence" => rule.precedence_sym&.id&.s_value }
     end
-    unknown = ::Arpaka::RubyGrammarActions::ACTIONS.keys - inventory.map { |rule| rule.fetch("id") }
-    warn "Arpaka: Unknown action IDs: #{unknown.inspect}" unless unknown.empty?
-    [grammar, inventory]
+    inventory
+  end
+
+  # Ruby's id.def is Ruby code. Evaluate it in a Box so its temporary locals,
+  # methods and constants cannot affect Arpaka or Lrama's process.
+  def self.preprocess(source, id_def)
+    box = ::Ruby::Box.new
+    ids = box.eval(File.read(id_def))
+    tokens = ids.fetch(:token_op).each_with_object({}) do |(_id, _op, token, id), result|
+      result[token] = id if token
+    end
+    return source unless tokens.any?
+
+    names = tokens.keys.map { |name| Regexp.escape(name) }.join("|")
+    token_pattern = /\bRUBY_TOKEN\((#{names})\)\s*(?=\s)/
+    source.lines.map do |line|
+      if line.start_with?("%token")
+        line.gsub(token_pattern) { tokens.fetch(Regexp.last_match(1)).to_s }
+      else
+        line
+      end
+    end.join
+  rescue KeyError, TypeError, SyntaxError => error
+    raise ::Arpaka::Error, "Ruby id.def preprocessing failed: #{error.message}"
+  end
+
+  def self.preprocess_names(source)
+    source.gsub(/\bRUBY_TOKEN\(([A-Za-z_][A-Za-z0-9_]*)\)/, '\\1')
   end
 
   def self.symbol_names(grammar)
@@ -69,12 +102,18 @@ module Arpaka::RubyGrammar
     end
   end
 
-  def self.artifacts(ruby_source:)
-    grammar, inventory = load_upstream(ruby_source)
+  def self.artifacts(ruby_source: nil, parse_y: nil)
+    grammar, inventory = if ruby_source
+      load_upstream(ruby_source)
+    else
+      load_parse_y(parse_y)
+    end
     names = symbol_names(grammar)
     name = ->(symbol) { symbol.term ? symbol.id.s_value : names.fetch(symbol.id.s_value) }
+    profile = ::Arpaka::Profile::RUBY.select(grammar)
+    expected_conflicts = (grammar.expect || 0) + profile.fetch("additional_expected_conflicts", 0)
     lines = ["/* Generated from ruby/ruby #{REVISION}. See Arpaka Action mappings. */",
-      "%no-stdlib", "%expect #{grammar.expect || 0}"]
+      "%no-stdlib", "%expect #{expected_conflicts}"]
     lines << "%define lr.type ielr" if grammar.ielr_defined?
     grammar.terms.each do |symbol|
       next if symbol.error_symbol? || symbol.undef_symbol?
@@ -100,7 +139,9 @@ module Arpaka::RubyGrammar
       else
         ""
       end
-      expression = ::Arpaka::RubyGrammarActions::ACTIONS[rule.id]&.last || "@builder.unsupported(#{rule.id})"
+      key = [rule.lhs.id.s_value, rule.rhs.map { |symbol| symbol.id.s_value }]
+      action = profile.fetch("actions")[rule.id] || profile.fetch("compatibility_actions", {})[key]
+      expression = action&.last || "@builder.unsupported(#{rule.id})"
       lines << "/* upstream parse.y:#{metadata.fetch(rule.id).fetch('line')}: #{rule.as_comment} */"
       lines << "#{name.call(rule.lhs)}: #{rhs}#{precedence} { $$ = #{expression} };"
     end
@@ -112,9 +153,13 @@ module Arpaka::RubyGrammar
     compact = rules.map do |rule|
       { id: rule.id, rule: rule.as_comment,
         line: metadata.fetch(rule.id).fetch("line"),
-        status: ::Arpaka::RubyGrammarActions::ACTIONS[rule.id]&.first || "unsupported" }
+        status: (profile.fetch("actions")[rule.id] ||
+          profile.fetch("compatibility_actions", {})[[rule.lhs.id.s_value,
+            rule.rhs.map { |symbol| symbol.id.s_value }]])&.first || "unsupported" }
     end
-    { "parse.y" => source, "rules.json" => JSON.pretty_generate(compact) + "\n" }
+    { "parse.y" => source, "rules.json" => JSON.pretty_generate(compact) + "\n",
+      "runtime" => profile.fetch("runtime", {}),
+      "source_profile" => profile.fetch("source_tag", REVISION) }
   end
 
   def self.verify_structure(original, source, names)
@@ -127,7 +172,7 @@ module Arpaka::RubyGrammar
       end.sort_by(&:inspect)
     end
     expected_rules = signature.call(original, names).map do |rule|
-      if rule[0] == names.fetch("value_expr_command") && rule[1] == [names.fetch("command")]
+      if names.key?("value_expr_command") && rule[0] == names.fetch("value_expr_command") && rule[1] == [names.fetch("command")]
         rule[0, 2] + ["tLBRACE_ARG"]
       else
         rule

data/lib/arpaka/runtime/ast.rb.erb

@@ -6,6 +6,14 @@ module AST
   If = Data.define(:condition, :then_body, :else_body)
   LocalRead = Data.define(:name)
   LocalWrite = Data.define(:name, :value)
+<% if runtime.fetch("ast_nodes", []).include?("assignment") -%>
+  Assignment = Data.define(:target, :value)
+  OpAssign = Data.define(:target, :operator, :value)
+  MultipleAssign = Data.define(:targets, :value)
+  Field = Data.define(:receiver, :operator, :name)
+  Alias = Data.define(:new_name, :old_name)
+  Undef = Data.define(:names)
+<% end -%>
   BareCall = Data.define(:name)
   ArrayLiteral = Data.define(:elements)
   HashLiteral = Data.define(:pairs)
@@ -26,10 +34,23 @@ module AST
   XStringLiteral = Data.define(:value)
   For = Data.define(:variable, :enumerable, :body)
   Def = Data.define(:name, :arguments, :body)
+<% if runtime.fetch("ast_nodes", []).include?("singleton_method") -%>
+  SingletonMethodDef = Data.define(:receiver, :name, :arguments, :body)
+<% end -%>
   ClassDef = Data.define(:name, :superclass, :body)
   ModuleDef = Data.define(:name, :body)
+<% if runtime.fetch("ast_nodes", []).include?("singleton_class") -%>
+  SingletonClassDef = Data.define(:receiver, :body)
+<% end -%>
   When = Data.define(:patterns, :body)
   Case = Data.define(:expression, :whens, :else_body)
+<% if runtime.fetch("ast_nodes", []).include?("pattern") -%>
+  Pattern = Data.define(:kind, :values)
+  RescueClause = Data.define(:exceptions, :variable, :body)
+  FullRescue = Data.define(:body, :clauses, :else_body, :ensure_body)
+  BeginBlock = Data.define(:body)
+  EndBlock = Data.define(:body)
+<% end -%>
   Lambda = Data.define(:arguments, :body)
   Sequence = Data.define(:statements)
 end

data/lib/arpaka/runtime/builder.rb.erb

@@ -84,6 +84,12 @@ class Builder
   def def_node(name, arguments, body)
     AST::Def.new(name.to_sym, (arguments || []).freeze, body || [].freeze)
   end
+<% if runtime.fetch("builder_methods", []).include?("singleton_method") -%>
+  def singleton_method_node(receiver, name, arguments, body)
+    AST::SingletonMethodDef.new(receiver, name.to_sym, (arguments || []).freeze,
+      body_nodes(body))
+  end
+<% end -%>
 
   def class_node(name, superclass, body)
     AST::ClassDef.new(name.to_sym, superclass, body_nodes(body))
@@ -92,6 +98,11 @@ class Builder
   def module_node(name, body)
     AST::ModuleDef.new(name.to_sym, body_nodes(body))
   end
+<% if runtime.fetch("builder_methods", []).include?("singleton_class") -%>
+  def singleton_class_node(receiver, body)
+    AST::SingletonClassDef.new(receiver, body_nodes(body))
+  end
+<% end -%>
 
   def body_nodes(body)
     value = body
@@ -170,6 +181,42 @@ class Builder
   def lambda_node(arguments, body)
     AST::Lambda.new((arguments || []).freeze, body_nodes(body))
   end
+<% if runtime.fetch("builder_methods", []).include?("rescue") -%>
+  def body_statement(body, clauses, else_body, ensure_body)
+    clauses = clauses.nil? ? [] : Array(clauses)
+    if clauses.empty? && else_body.nil? && ensure_body.nil?
+      body_nodes(body)
+    else
+      full_rescue(body, clauses, else_body, ensure_body)
+    end
+  end
+
+  def rescue_chain(clause, rest)
+    [clause] + (rest.nil? ? [] : Array(rest))
+  end
+
+  def pattern(kind, values)
+    AST::Pattern.new(kind.to_sym, Array(values).freeze)
+  end
+
+  def rescue_clause(exceptions, variable, body)
+    AST::RescueClause.new(Array(exceptions).freeze, variable, body_nodes(body))
+  end
+
+  def full_rescue(body, clauses, else_body, ensure_body)
+    AST::FullRescue.new(body_nodes(body), Array(clauses).freeze,
+      else_body.nil? ? nil : body_nodes(else_body),
+      ensure_body.nil? ? nil : body_nodes(ensure_body))
+  end
+
+  def begin_block(body)
+    AST::BeginBlock.new(body_nodes(body))
+  end
+
+  def end_block(body)
+    AST::EndBlock.new(body_nodes(body))
+  end
+<% end -%>
 
   def unary(operator, operand)
     AST::Unary.new(operator, operand)
@@ -212,6 +259,35 @@ class Builder
   def assign(name, value)
     AST::LocalWrite.new(name, value)
   end
+<% if runtime.fetch("builder_methods", []).include?("assignment") -%>
+  def assignment(name, value)
+    if name.is_a?(AST::Field) || name.is_a?(AST::Index)
+      AST::Assignment.new(name, value)
+    else
+      AST::LocalWrite.new(name, value)
+    end
+  end
+
+  def opassign(target, operator, value)
+    AST::OpAssign.new(target, operator.to_sym, value)
+  end
+
+  def multiple_assign(targets, value)
+    AST::MultipleAssign.new(Array(targets).freeze, value)
+  end
+
+  def field(receiver, operator, name)
+    AST::Field.new(receiver, operator.to_sym, name.to_sym)
+  end
+
+  def alias_node(new_name, old_name)
+    AST::Alias.new(new_name, old_name)
+  end
+
+  def undef_node(names)
+    AST::Undef.new(Array(names).freeze)
+  end
+<% end -%>
 
   def array(elements)
     AST::ArrayLiteral.new(elements.freeze)

data/lib/arpaka/runtime/lexer.rb.erb

@@ -1287,7 +1287,14 @@ class Lexer
         OP_TOKENS.fetch(text, :tOP_ASGN)
       end
       @context.lambda_pending = true if token == :tLAMBDA
+<% source_text_tokens = runtime.fetch("lexer", {}).fetch("source_text_tokens", []) -%>
+<% if source_text_tokens.empty? -%>
       return [token, nil]
+<% elsif source_text_tokens.one? -%>
+      return [token, token == <%= source_text_tokens.first.to_sym.inspect %> ? text : nil]
+<% else -%>
+      return [token, <%= source_text_tokens.map { |value| value.to_sym }.inspect %>.include?(token) ? text : nil]
+<% end -%>
     end
     value = byte.chr
     advance

data/lib/arpaka.rb

@@ -5,12 +5,12 @@ require_relative "lrama/ruby"
 module Arpaka
   class Error < Lrama::Ruby::Error; end
 
-  def self.generate(ruby_source:, class_name: "RubyParser")
-    Generator.new.generate(ruby_source: ruby_source, class_name: class_name)
+  def self.generate(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
+    Generator.new.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
   end
 
-  def self.compile(ruby_source:, class_name: "RubyParser")
-    source = generate(ruby_source: ruby_source, class_name: class_name)
+  def self.compile(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
+    source = generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
     box = ::Ruby::Box.new
     box.eval(source)
     box.const_get(class_name, false)

data/lib/lrama/ruby/version.rb

@@ -2,6 +2,6 @@
 
 module Lrama
   module Ruby
-    VERSION = "0.1.1"
+    VERSION = "1.0.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arpaka
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
 - MATSUMOTO, Katsuyoshi
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: racc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A Ruby parser frontend and Ruby output backend for Lrama.
 email:
 - github@katsyoshi.org
@@ -45,6 +59,9 @@ files:
 - lib/arpaka/actions.rb
 - lib/arpaka/cli.rb
 - lib/arpaka/generator.rb
+- lib/arpaka/profile/ruby.rb
+- lib/arpaka/profile/ruby_3_3.rb
+- lib/arpaka/profile/ruby_3_4.rb
 - lib/arpaka/ruby_grammar.rb
 - lib/arpaka/runtime/ast.rb.erb
 - lib/arpaka/runtime/builder.rb.erb