arpaka 1.0.0 → 2.0.0

data/lib/arpaka/ruby/runtime/structured.rb.erb

@@ -0,0 +1,111 @@
+module AST
+  Source = Data.define(:filename, :children)
+  Class = Data.define(:name, :superclass, :children)
+  Module = Data.define(:name, :children)
+  Method = Data.define(:name, :kind, :receiver, :children)
+
+  def self.project(detailed_ast, filename:, modifier_nodes: nil)
+    Source.new(filename, definitions_in(detailed_ast, modifier_nodes: modifier_nodes).freeze)
+  end
+
+  def self.definitions_in(value, modifier_nodes:, method_kind: :instance, receiver: nil)
+    case value
+    when Array
+      value.flat_map do |item|
+        definitions_in(item, modifier_nodes: modifier_nodes,
+          method_kind: method_kind, receiver: receiver)
+      end
+    when AST::ClassDef
+      definitions_in([value.name, value.superclass], modifier_nodes: modifier_nodes,
+        method_kind: method_kind, receiver: receiver) +
+        [Class.new(static_name(value.name), static_name(value.superclass),
+          definitions_in(value.body, modifier_nodes: modifier_nodes).freeze)]
+    when AST::ModuleDef
+      definitions_in(value.name, modifier_nodes: modifier_nodes,
+        method_kind: method_kind, receiver: receiver) +
+        [Module.new(static_name(value.name),
+          definitions_in(value.body, modifier_nodes: modifier_nodes).freeze)]
+    when AST::Def
+      children = definitions_in(value.arguments, modifier_nodes: modifier_nodes,
+        method_kind: method_kind, receiver: receiver) +
+        definitions_in(value.body, modifier_nodes: modifier_nodes,
+          method_kind: method_kind, receiver: receiver)
+      [Method.new(value.name, method_kind, receiver, children.freeze)]
+    when AST::SingletonMethodDef
+      header = definitions_in(value.receiver, modifier_nodes: modifier_nodes,
+        method_kind: method_kind, receiver: receiver)
+      children = definitions_in(value.arguments, modifier_nodes: modifier_nodes,
+        method_kind: method_kind, receiver: receiver) +
+        definitions_in(value.body, modifier_nodes: modifier_nodes,
+          method_kind: method_kind, receiver: receiver)
+      header + [Method.new(value.name, :singleton, receiver_name(value.receiver),
+        children.freeze)]
+    when AST::SingletonClassDef
+      definitions_in(value.receiver, modifier_nodes: modifier_nodes,
+        method_kind: method_kind, receiver: receiver) +
+        definitions_in(value.body, modifier_nodes: modifier_nodes,
+          method_kind: :singleton, receiver: receiver_name(value.receiver))
+    when AST::If
+      members = if modifier_nodes&.key?(value)
+        [value.then_body, value.condition, value.else_body]
+      else
+        [value.condition, value.then_body, value.else_body]
+      end
+      members.flat_map do |member|
+        definitions_in(member, modifier_nodes: modifier_nodes,
+          method_kind: method_kind, receiver: receiver)
+      end
+    when AST::Loop
+      members = if modifier_nodes&.key?(value)
+        [value.body, value.condition]
+      else
+        [value.condition, value.body]
+      end
+      members.flat_map do |member|
+        definitions_in(member, modifier_nodes: modifier_nodes,
+          method_kind: method_kind, receiver: receiver)
+      end
+    else
+      return [] unless ast_node?(value)
+
+      value.members.flat_map do |member|
+        definitions_in(value.public_send(member), modifier_nodes: modifier_nodes,
+          method_kind: method_kind, receiver: receiver)
+      end
+    end
+  end
+  private_class_method :definitions_in
+
+  def self.ast_node?(value)
+    value.is_a?(Data) && value.class.name&.start_with?("#{AST.name}::")
+  end
+  private_class_method :ast_node?
+
+  def self.static_name(value)
+    case value
+    when Symbol
+      value
+    when AST::Variable
+      value.name if value.kind == :constant
+    when AST::AbsoluteConstant
+      "::#{value.name}"
+    when AST::ConstantPath
+      receiver = static_name(value.receiver)
+      "#{receiver}::#{value.name}" if receiver
+    end
+  end
+  private_class_method :static_name
+
+  def self.receiver_name(value)
+    return :self if defined?(AST::Self) && value.is_a?(AST::Self)
+    return :self if value.is_a?(AST::Literal) && value.value == :self
+
+    case value
+    when AST::LocalRead, AST::BareCall
+      value.name
+    else
+      static_name(value)
+    end
+  end
+  private_class_method :receiver_name
+end

data/lib/arpaka/ruby.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "ruby/class_name_parser"
+
+module Arpaka
+  module Ruby
+    def self.generate(ruby_source: nil, parse_y: nil, class_name: "RubyParser", ast_type: :detailed)
+      Generator.new.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name,
+        ast_type: ast_type)
+    end
+
+    def self.compile(ruby_source: nil, parse_y: nil, class_name: "RubyParser", ast_type: :detailed)
+      class_name = ClassNameParser.parse(class_name)
+      source = generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name.to_s,
+        ast_type: ast_type)
+      box = ::Ruby::Box.new
+      box.eval(source)
+      class_name.resolve(box)
+    end
+  end
+end
+
+require_relative "ruby/generator"

data/lib/arpaka.rb

@@ -5,16 +5,13 @@ require_relative "lrama/ruby"
 module Arpaka
   class Error < Lrama::Ruby::Error; end
 
-  def self.generate(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
-    Generator.new.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
+  require_relative "arpaka/ruby"
+
+  def self.generate(ruby_source: nil, parse_y: nil, class_name: "RubyParser", ast_type: :detailed)
+    Ruby.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name, ast_type: ast_type)
   end
 
-  def self.compile(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
-    source = generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
-    box = ::Ruby::Box.new
-    box.eval(source)
-    box.const_get(class_name, false)
+  def self.compile(ruby_source: nil, parse_y: nil, class_name: "RubyParser", ast_type: :detailed)
+    Ruby.compile(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name, ast_type: ast_type)
   end
 end
-
-require_relative "arpaka/generator"

data/lib/lrama/ruby/backend.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "lrama"
-require "erb"
+require_relative "parser_emitter"
 
 module Lrama
   module Ruby
@@ -30,15 +30,10 @@ module Lrama
         tokens = token_names(grammar)
         actions = grammar.rules.filter_map do |rule|
           next if recognizer || !rule.token_code
-          # Preserve whitespace inside multiline strings and heredocs.
-          "    when #{rule.id + 1} # #{rule.as_comment.gsub(/[\r\n]/, ' ')}\n#{translate_action(rule)}\n"
+          "    when #{rule.id + 1}\n#{translate_action(rule)}\n"
         end.join
-        template = File.read(File.expand_path("parser.rb.erb", __dir__))
-        result = ::ERB.new(template, trim_mode: "-").result_with_hash(
-          class_name: class_name, context: context, tokens: tokens, actions: actions,
-          recognizer: recognizer
-        )
-        result
+        ParserEmitter.new.emit(class_name: class_name, context: context, tokens: tokens,
+          actions: actions, recognizer: recognizer)
       end
 
       private

data/lib/lrama/ruby/parser.rb.erb

@@ -1,11 +1,37 @@
 # frozen_string_literal: true
 
 # Generated by Arpaka/Lrama. Do not edit.
+<% if compressed_tables -%>
+require "zlib"
+<% end -%>
 <% if recognizer -%>
 # Recognition only: C actions and support code are omitted.
 # Supply pre-tokenized input; acceptance returns true. No error recovery.
 <% end -%>
 class <%= class_name %>
+<% if compressed_tables -%>
+  def self.__decode_table(encoded, length)
+    bytes = Zlib::Inflate.inflate(encoded.unpack1("m0")).bytes
+    values = []
+    value = 0
+    shift = 0
+    bytes.each do |byte|
+      value |= (byte & 0x7f) << shift
+      if byte < 0x80
+        values << (value.even? ? value / 2 : -(value / 2) - 1)
+        value = 0
+        shift = 0
+      else
+        shift += 7
+      end
+    end
+    raise ArgumentError, "invalid parser table encoding" unless shift.zero? && values.length == length
+
+    values.freeze
+  end
+  private_class_method :__decode_table
+<% end -%>
+
   class ParseError < StandardError
     attr_reader :token, :state
 
@@ -16,15 +42,6 @@ class <%= class_name %>
     end
   end
 
-  TOKENS = <%= tokens.inspect %>.freeze
-<% %i[yytranslate yypact yydefact yypgoto yydefgoto yytable yycheck yyr1 yyr2].each do |name| -%>
-  <%= name.upcase %> = <%= context.public_send(name).inspect %>.freeze
-<% end -%>
-  YYFINAL = <%= context.yyfinal %>
-  YYNTOKENS = <%= context.yyntokens %>
-  YYPACT_NINF = <%= context.yypact_ninf %>
-  YYTABLE_NINF = <%= context.yytable_ninf %>
-
   attr_accessor :lexical_context
   attr_reader :parse_trace, :lookahead_token, :lookahead_value, :lookahead_state
 
@@ -124,4 +141,13 @@ class <%= class_name %>
 <% end -%>
     yyval
   end
+
+  TOKENS = <%= tokens.inspect %>.freeze
+<% %i[yytranslate yypact yydefact yypgoto yydefgoto yytable yycheck yyr1 yyr2].each do |name| -%>
+  <%= name.upcase %> = <%= tables.fetch(name) %>
+<% end -%>
+  YYFINAL = <%= context.yyfinal %>
+  YYNTOKENS = <%= context.yyntokens %>
+  YYPACT_NINF = <%= context.yypact_ninf %>
+  YYTABLE_NINF = <%= context.yytable_ninf %>
 end

data/lib/lrama/ruby/parser_emitter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "erb"
+require_relative "table_codec"
+
+module Lrama
+  module Ruby
+    class ParserEmitter
+      def emit(class_name:, context:, tokens:, actions:, recognizer:)
+        template = File.read(File.expand_path("parser.rb.erb", __dir__))
+        names = %i[yytranslate yypact yydefact yypgoto yydefgoto yytable yycheck yyr1 yyr2]
+        literal_tables = names.to_h do |name|
+          [name, "#{context.public_send(name).inspect}.freeze"]
+        end
+        candidates = names.to_h do |name|
+          values = context.public_send(name)
+          compressed = TableCodec.compress(values)
+          encoded = "__decode_table(#{compressed.inspect}, #{values.length})"
+          [name, encoded.bytesize < literal_tables.fetch(name).bytesize ? encoded : literal_tables.fetch(name)]
+        end
+
+        literal_source = render(template, class_name:, context:, tokens:, actions:, recognizer:,
+          tables: literal_tables, compressed_tables: false)
+        return literal_source unless candidates.values.any? { |expression| expression.start_with?("__decode_table(") }
+
+        compressed_source = render(template, class_name:, context:, tokens:, actions:, recognizer:,
+          tables: candidates, compressed_tables: true)
+        compressed_source.bytesize < literal_source.bytesize ? compressed_source : literal_source
+      end
+
+      private
+
+      def render(template, class_name:, context:, tokens:, actions:, recognizer:, tables:, compressed_tables:)
+        ::ERB.new(template, trim_mode: "-").result_with_hash(
+          class_name: class_name, context: context, tokens: tokens, actions: actions,
+          recognizer: recognizer, tables: tables, compressed_tables: compressed_tables
+        )
+      end
+    end
+  end
+end

data/lib/lrama/ruby/table_codec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module Lrama
+  module Ruby
+    module TableCodec
+      module_function
+
+      def encode(values)
+        bytes = []
+        values.each do |value|
+          raise TypeError, "table values must be integers" unless value.is_a?(Integer)
+
+          unsigned = value.negative? ? (-value * 2) - 1 : value * 2
+          while unsigned >= 0x80
+            bytes << ((unsigned & 0x7f) | 0x80)
+            unsigned >>= 7
+          end
+          bytes << unsigned
+        end
+        bytes.pack("C*").force_encoding(Encoding::BINARY)
+      end
+
+      def compress(values)
+        [Zlib::Deflate.deflate(encode(values))].pack("m0")
+      end
+
+      def decode(encoded)
+        bytes = Zlib::Inflate.inflate(encoded.unpack1("m0")).bytes
+        values = []
+        value = 0
+        shift = 0
+        bytes.each do |byte|
+          value |= (byte & 0x7f) << shift
+          if byte < 0x80
+            values << (value.even? ? value / 2 : -(value / 2) - 1)
+            value = 0
+            shift = 0
+          else
+            shift += 7
+          end
+        end
+        raise ArgumentError, "invalid parser table encoding" unless shift.zero?
+
+        values.freeze
+      end
+    end
+  end
+end

data/lib/lrama/ruby/version.rb

@@ -2,6 +2,6 @@
 
 module Lrama
   module Ruby
-    VERSION = "1.0.0"
+    VERSION = "2.0.0"
   end
 end

data/lib/lrama/ruby.rb

@@ -26,3 +26,4 @@ module Lrama
 end
 
 require_relative "ruby/generator"
+

data/sig/arpaka.rbs

@@ -3,12 +3,17 @@ module Arpaka
     def statements: () -> Array[untyped]
   end
 
+  interface _StructuredSource
+    def filename: () -> String
+    def children: () -> Array[untyped]
+  end
+
   interface _Frontend
-    def parse: (String source, ?filename: String) -> _Program
+    def parse: (String source, ?filename: String) -> (_Program | _StructuredSource)
   end
 
-  def self.generate: (ruby_source: String, ?class_name: String) -> String
-  def self.compile: (ruby_source: String, ?class_name: String) -> _Frontend
+  def self.generate: (?ruby_source: String?, ?parse_y: String?, ?class_name: String, ?ast_type: (:detailed | :structured | String)) -> String
+  def self.compile: (?ruby_source: String?, ?parse_y: String?, ?class_name: String, ?ast_type: (:detailed | :structured | String)) -> _Frontend
 
   class Error < ::Lrama::Ruby::Error
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arpaka
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - MATSUMOTO, Katsuyoshi
@@ -50,31 +50,47 @@ files:
 - README.md
 - Rakefile
 - assets/arpaka-logo.svg
+- docs/architecture.dot
+- docs/architecture.md
+- docs/architecture.webp
+- docs/contributes.md
+- docs/usage.md
 - examples/calculator.y
 - examples/calculator_ast.y
 - exe/arpaka
 - lib/arpaka.rb
-- lib/arpaka/BSDL
-- lib/arpaka/COPYING
-- lib/arpaka/actions.rb
 - lib/arpaka/cli.rb
-- lib/arpaka/generator.rb
-- lib/arpaka/profile/ruby.rb
-- lib/arpaka/profile/ruby_3_3.rb
-- lib/arpaka/profile/ruby_3_4.rb
-- lib/arpaka/ruby_grammar.rb
-- lib/arpaka/runtime/ast.rb.erb
-- lib/arpaka/runtime/builder.rb.erb
-- lib/arpaka/runtime/frontend.rb.erb
-- lib/arpaka/runtime/lexer.rb.erb
-- lib/arpaka/source.json
-- lib/arpaka/upstream_rules.json
+- lib/arpaka/plugins/cli.rb
+- lib/arpaka/plugins/sample.rb
+- lib/arpaka/plugins/sample_runner.rb
+- lib/arpaka/ruby.rb
+- lib/arpaka/ruby/BSDL
+- lib/arpaka/ruby/COPYING
+- lib/arpaka/ruby/actions.rb
+- lib/arpaka/ruby/class_name_parser.rb
+- lib/arpaka/ruby/cli.rb
+- lib/arpaka/ruby/frontend_emitter.rb
+- lib/arpaka/ruby/generator.rb
+- lib/arpaka/ruby/grammar.rb
+- lib/arpaka/ruby/grammar_emitter.rb
+- lib/arpaka/ruby/profile.rb
+- lib/arpaka/ruby/profiles/ruby_3_3.rb
+- lib/arpaka/ruby/profiles/ruby_3_4.rb
+- lib/arpaka/ruby/runtime/detailed/ast.rb.erb
+- lib/arpaka/ruby/runtime/detailed/builder.rb.erb
+- lib/arpaka/ruby/runtime/frontend.rb.erb
+- lib/arpaka/ruby/runtime/lexer.rb.erb
+- lib/arpaka/ruby/runtime/structured.rb.erb
+- lib/arpaka/ruby/source.json
+- lib/arpaka/ruby/upstream_rules.json
 - lib/lrama/ruby.rb
 - lib/lrama/ruby/action_code.rb
 - lib/lrama/ruby/action_scanner.rb
 - lib/lrama/ruby/backend.rb
 - lib/lrama/ruby/generator.rb
 - lib/lrama/ruby/parser.rb.erb
+- lib/lrama/ruby/parser_emitter.rb
+- lib/lrama/ruby/table_codec.rb
 - lib/lrama/ruby/version.rb
 - sig/arpaka.rbs
 - sig/lrama/ruby.rbs
@@ -97,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.20
+rubygems_version: 4.0.24
 specification_version: 4
 summary: Ruby language parser frontend and Lrama output backend
 test_files: []

data/lib/arpaka/generator.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-require "erb"
-require "json"
-require_relative "ruby_grammar"
-
-module Arpaka
-  class Generator
-    def generate(ruby_source: nil, parse_y: nil, class_name:)
-      if ruby_source.nil? == parse_y.nil?
-        raise Error, "specify exactly one of ruby_source or parse_y"
-      end
-      unless /\A[A-Z][a-zA-Z0-9_]*\z/.match?(class_name)
-        raise Error, "class_name must be a single Ruby constant name"
-      end
-      unless ::Ruby::Box.enabled?
-        raise Error, "Ruby Box is required for generation. Start Ruby with RUBY_BOX=1."
-      end
-      artifacts = RubyGrammar.artifacts(ruby_source: ruby_source, parse_y: parse_y)
-      grammar_filename = parse_y || File.join(ruby_source, "parse.y")
-      parser = Lrama::Ruby.generate(artifacts.fetch("parse.y"),
-        filename: grammar_filename, class_name: "Parser", allow_error_rules: true)
-      parser = parser.delete_prefix("# frozen_string_literal: true\n")
-      rules = JSON.parse(artifacts.fetch("rules.json"))
-      runtime_options = artifacts.fetch("runtime", {})
-      runtime = %w[frontend ast builder lexer].map do |name|
-        template = File.read(File.join(__dir__, "runtime", "#{name}.rb.erb"))
-        ERB.new(template, trim_mode: "-").result_with_hash(rules: rules, runtime: runtime_options)
-      end.join("\n")
-      notices = [File.read(File.expand_path("../../LICENSE.txt", __dir__))]
-      %w[COPYING BSDL].each do |name|
-        path = ruby_source ? File.join(ruby_source, name) : File.join(__dir__, name)
-        notices << File.read(path)
-      end
-      source = "# frozen_string_literal: true\n" \
-        "# Generated by Arpaka #{Lrama::Ruby::VERSION}. Do not edit.\n" \
-        "# Ruby source profile: #{ruby_source ? artifacts.fetch('source_profile') : 'parse.y only'}\n" \
-        "# parse.y SHA256: #{Digest::SHA256.file(grammar_filename).hexdigest}\n" \
-        "#{notices.join("\n").lines.map { |line| "# #{line}" }.join}\n" \
-        "class #{class_name}\n#{parser}\n#{runtime}\nend\n"
-      # Compile in an isolated box to validate syntax without executing actions.
-      box = ::Ruby::Box.new
-      box.eval(source)
-      source
-    rescue SyntaxError => error
-      raise Error, "Invalid generated frontend: #{error.message}"
-    rescue SystemCallError => error
-      raise Error, "Cannot read Ruby frontend inputs: #{error.message}"
-    end
-  end
-end

data/lib/arpaka/profile/ruby.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Arpaka
-  module Profile
-    module RUBY
-      CURRENT = {
-        "name" => "ruby-4.0",
-        "actions" => ::Arpaka::RubyGrammarActions::ACTIONS,
-        "additional_expected_conflicts" => 0,
-        "runtime" => {}.freeze
-      }.freeze
-
-      def self.select(grammar)
-        if grammar.rules.any? { |rule| rule.lhs.id.s_value == "value_expr_command" }
-          CURRENT
-        elsif grammar.rules.any? { |rule| rule.lhs.id.s_value == "top_compstmt" &&
-          rule.rhs.any? { |symbol| symbol.id.s_value == "option_terms" } }
-          legacy
-        elsif grammar.rules.any? { |rule| rule.lhs.id.s_value == "top_compstmt" &&
-          rule.rhs.any? { |symbol| symbol.id.s_value == "opt_terms" } }
-          legacy_3_3
-        else
-          raise ::Arpaka::Error, "Unsupported Ruby parse.y profile"
-        end
-      end
-
-      def self.legacy
-        @legacy ||= load("ruby_3_4.rb", "ruby-3.4")
-      end
-
-      def self.legacy_3_3
-        @legacy_3_3 ||= load("ruby_3_3.rb", "ruby-3.3")
-      end
-
-      def self.load(filename, name)
-        box = ::Ruby::Box.new
-        profile = box.eval(File.read(File.join(__dir__, filename)))
-        unless profile.is_a?(Hash) && profile["name"] == name
-          raise ::Arpaka::Error, "Invalid #{name} profile"
-        end
-        profile.freeze
-      rescue SyntaxError, TypeError => error
-        raise ::Arpaka::Error, "Ruby profile loading failed: #{error.message}"
-      end
-    end
-  end
-end