peruby 0.1.0

data/lib/peruby/cli.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'pp'
+
+module Peruby
+  # Command-line entry point for the interpreter.
+  class CLI
+    def self.run(argv, input: $stdin, out: $stdout, err: $stderr)
+      new(argv, input:, out:, err:).run
+    end
+
+    def initialize(argv, input:, out:, err:)
+      @argv = argv
+      @input = input
+      @out = out
+      @err = err
+      @debug = ENV.fetch('PERUBY_DEBUG', '').split(',')
+    end
+
+    def run
+      parser.order!(@argv)
+      return 0 if @shown_version
+      return dump_tokens if @dump_tokens
+      return dump_ast if @dump_ast
+      return dump_ops if @dump_ops
+
+      execute
+    rescue OptionParser::ParseError => e
+      @err.puts e.message
+      1
+    rescue CompileError => e
+      @err.puts e.message
+      @err.puts "Execution of #{@file} aborted due to compilation errors." if @file
+      1
+    rescue PerlExit => e
+      status = @runtime && @runtime.stash.glob('?').scalar.get
+      status.nil? ? e.status : Conv.to_num(status).to_i
+    rescue PerlError => e
+      @err.print(e.formatted? ? e.message : @runtime.error_message(e.message, file: @file))
+      1
+    end
+
+    private
+
+    # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
+    def parser
+      # rubocop:disable-next Metrics/BlockLength
+      OptionParser.new do |options|
+        options.banner = 'Usage: peruby [options] [programfile]'
+        options.on('-v', '--version', 'Print the version') do
+          @out.puts "peruby #{VERSION}"
+          @shown_version = true
+        end
+        options.on('--dump-tokens', 'Print lexer tokens') { @dump_tokens = true }
+        options.on('--dump-ast', 'Print the syntax tree') { @dump_ast = true }
+        options.on('--dump-ops', 'Print the operation tree') { @dump_ops = true }
+        options.on('--trace-ops', 'Trace operation execution') do
+          @trace_ops = true
+          @interpreter = :tree
+        end
+        options.on('-e CODE', 'Execute one line of program') { |code| @code = code }
+        options.on('-E CODE', 'Execute one line with features enabled') { |code| @code = code }
+        options.on('-c', 'Check syntax only') { @check = true }
+        options.on('-n', 'Run the program for each input record') { @loop = true }
+        options.on('-p', 'Run and print each input record') { @loop = @print_loop = true }
+        options.on('-l', 'Chomp input and append a newline to print') { @line_end = true }
+        options.on('-a', 'Autosplit input into @F') { @autosplit = true }
+        options.on('-F PATTERN', 'Autosplit pattern') do |pattern|
+          @split_pattern = pattern
+          @autosplit = true
+        end
+        options.on('-0[OCTAL]', 'Set the input record separator') { |octal| @separator = separator(octal) }
+        options.on('-M MODULE', 'Use a module before the program') { |name| (@use_modules ||= []) << name }
+        options.on('-m MODULE', 'Load a module without imports') { |name| (@load_modules ||= []) << name }
+        options.on('-s', 'Set script switches as package variables') { @script_switches = true }
+        options.on('-I DIRECTORY', 'Add a module search path') { |path| (@include_paths ||= []) << path }
+        options.on('--interpreter MODE', %w[compiled tree], 'Select compiled or tree execution') do |mode|
+          @interpreter = mode.to_sym
+        end
+        options.on('--refcount', 'Run DESTROY when tracked references are released') { @refcount = true }
+      end
+    end
+
+    def dump_tokens
+      @file = @argv.shift or raise CompileError, 'no program file supplied'
+      Lexer.new(File.read(@file), file: @file).each { |token| @out.puts token }
+      0
+    end
+
+    def dump_ast
+      @file = @argv.shift or raise CompileError, 'no program file supplied'
+      @out.puts Parser.parse(File.read(@file), file: @file).to_sexp.inspect
+      0
+    end
+
+    def dump_ops
+      @file = @argv.shift or raise CompileError, 'no program file supplied'
+      @runtime = Runtime.new(stdin: @input, stdout: @out, stderr: @err)
+      PP.pp(CompileUnit.new(runtime: @runtime, file: @file, interpreter: :tree).compile(File.read(@file)), @out)
+      0
+    ensure
+      @runtime&.shutdown
+    end
+
+    def execute
+      @file = @code ? '-e' : @argv.shift
+      raise CompileError, 'no program supplied' unless @file
+
+      source = module_prelude + (@code || File.read(@file))
+      debug_source(source)
+      @runtime = Runtime.new(stdin: @input, stdout: @out, stderr: @err, refcount: @refcount)
+      configure_runtime
+      unit = CompileUnit.new(runtime: @runtime, file: @file, interpreter: @interpreter || :compiled)
+      if @check
+        unit.compile(source)
+        @out.puts "#{@file} syntax OK"
+      else
+        operation = unit.compile(source)
+        @runtime.run_phase(:init)
+        run_operation(operation)
+      end
+      0
+    ensure
+      @runtime&.shutdown
+    end
+
+    def debug_source(source)
+      Lexer.new(source, file: @file).each { |token| @err.puts "lexer #{token}" } if @debug.include?('lexer')
+      @err.puts "parser #{Parser.parse(source, file: @file).to_sexp.inspect}" if @debug.include?('parser')
+    end
+
+    def run_operation(operation)
+      execute = -> { @loop ? run_input_loop(operation) : operation.run(@runtime.env, :void) }
+      return execute.call unless @trace_ops
+
+      trace = TracePoint.new(:call) do |event|
+        next unless event.method_id == :run && event.self.is_a?(Op::Base)
+
+        @err.puts "run #{event.self.class.name.delete_prefix('Peruby::Op::')}"
+      end
+      trace.enable(&execute)
+    end
+
+    def configure_runtime
+      Array(@include_paths).reverse_each do |path|
+        @runtime.stash.glob('INC').array.cells.unshift(Scalar.new(path))
+      end
+      Array(@load_modules).each { |name| load_module(name) }
+      configure_arguments
+      @runtime.stash.glob('/').scalar.set(record_separator)
+      @runtime.stash.glob('\\').scalar.set("\n") if @line_end
+    end
+
+    def configure_arguments
+      if @script_switches
+        @argv.delete_if do |argument|
+          next false unless argument.match?(/\A-[A-Za-z_]\w*(?:=.*)?\z/)
+
+          name, value = argument.delete_prefix('-').split('=', 2)
+          @runtime.stash.glob(name).scalar.set(value || 1)
+          true
+        end
+      end
+      cells = @argv.map { |argument| Scalar.new(argument) }
+      @runtime.stash.glob('ARGV').array.cells.replace(cells)
+    end
+
+    def load_module(name)
+      if Op::Use::CORE.include?(name)
+        @runtime.stash.glob('INC').hash.slot("#{name.gsub('::', '/')}.pm").set('(peruby internal)')
+      else
+        @runtime.module_loader.require_module(name)
+      end
+    end
+
+    def module_prelude
+      Array(@use_modules).map do |specification|
+        name, imports = specification.split('=', 2)
+        imports ? "use #{name} qw(#{imports.tr(',', ' ')});\n" : "use #{name};\n"
+      end.join
+    end
+
+    def run_input_loop(operation)
+      each_input_record do |record|
+        record = record.delete_suffix(record_separator.to_s) if @line_end && record_separator
+        @runtime.stash.glob('_').scalar.set(record)
+        autosplit(record) if @autosplit
+        operation.run(@runtime.env, :void)
+        @out.print(Conv.to_str(@runtime.stash.glob('_').scalar.get), @line_end ? "\n" : '') if @print_loop
+      end
+    end
+
+    def each_input_record(&block)
+      return each_record(@input, &block) if @argv.empty?
+
+      @argv.each do |path|
+        File.open(path) { |input| each_record(input, &block) }
+      end
+    end
+
+    def each_record(input, &block)
+      return block.call(input.read) unless record_separator
+
+      input.each_line(record_separator, &block)
+    end
+
+    def autosplit(record)
+      fields = @split_pattern ? record.split(RegexpCompiler.compile(@split_pattern)) : record.split
+      @runtime.stash.glob('F').array.cells.replace(fields.map { |field| Scalar.new(field) })
+    end
+
+    def record_separator
+      defined?(@separator) ? @separator : "\n"
+    end
+
+    def separator(octal)
+      value = octal || '0'
+      return nil if value == '777'
+
+      value.to_i(8).chr
+    end
+  end
+end

data/lib/peruby/compile_unit.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Peruby
+  # One parse/compile/run unit.
+  class CompileUnit
+    attr_reader :runtime, :file
+
+    def initialize(runtime: Runtime.new, file: '-e', interpreter: :compiled)
+      @runtime = runtime
+      @file = file
+      @interpreter = interpreter
+    end
+
+    def compile(source, env: nil)
+      @runtime.with_file(@file) do
+        previous_env = @compile_env
+        @compile_env = (env || @runtime.env).fork
+        @compile_scopes = [{}]
+        @predeclared = {}.compare_by_identity
+        strict = @compile_env.strict_categories
+        warnings = @runtime.warning_categories
+        tree = Parser.parse(source, file: @file, known_subs: known_subs(@compile_env.package), unit: self,
+                                    package: @compile_env.package)
+        Validator.validate(tree, runtime: @runtime, strict:, warnings:)
+        operation = compiler.compile(tree)
+        @runtime.run_phase(:check)
+        @interpreter == :tree ? operation : Op::Compiled.new(operation.to_callable, operation.to_subroutine_callable)
+      ensure
+        @compile_env = previous_env
+      end
+    end
+
+    def run(source, context: :void)
+      @runtime.with_file(@file) do
+        operation = compile(source)
+        @runtime.run_phase(:init)
+        @runtime.local_stack.within { operation.run(@runtime.env, context) }
+      end
+    end
+
+    def register_phase(kind, block, _token, package)
+      Validator.validate(block, runtime: @runtime, strict: @compile_env&.strict_categories || Set.new,
+                                warnings: @runtime.warning_categories)
+      operation = Op::PackageBlock.new(package, compiler.compile(block))
+      env = phase_env
+      return operation.run(env, :void) if kind == :begin
+
+      @runtime.register_phase(kind, operation, env:)
+    end
+
+    def run_at_compile_time(node)
+      env = node.is_a?(Node::Use) ? @compile_env : phase_env
+      compiler.compile(node).run(env, :void)
+    end
+
+    def enter_compile_scope
+      @compile_scopes << {}
+    end
+
+    def leave_compile_scope
+      @compile_scopes.pop
+    end
+
+    def predeclare(node)
+      # perlmod: BEGIN runs immediately and can see lexicals declared before it.
+      variables = node.variable.is_a?(Node::Variable) ? [node.variable] : node.variable.items
+      values = variables.map { |variable| empty_value(variable.sigil) }
+      variables.zip(values) do |variable, value|
+        @compile_scopes.last[[variable.sigil, variable.name]] = value if value
+      end
+      @predeclared[node] = values
+    end
+
+    def predeclare_variable(variable)
+      @compile_scopes.last[[variable.sigil, variable.name]] = empty_value(variable.sigil)
+    end
+
+    private
+
+    def compiler = Compiler.new(predeclared: @predeclared)
+
+    def phase_env
+      @compile_env.fork.tap do |env|
+        @compile_scopes.each do |scope|
+          scope.each { |(sigil, name), value| env.bind(sigil, name, value) }
+        end
+      end
+    end
+
+    def empty_value(sigil)
+      { scalar: Scalar, array: PerlArray, hash: PerlHash }[sigil]&.new
+    end
+
+    def known_subs(package)
+      @runtime.stash.symbols(package).filter_map do |name, glob|
+        next unless glob.code
+
+        prototype = glob.code.prototype
+        [name, prototype&.start_with?('&') ? :listop : prototype]
+      end.to_h
+    end
+  end
+end

data/lib/peruby/compiler.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require_relative 'op/literal'
+require_relative 'op/variable'
+require_relative 'op/expression'
+require_relative 'op/assign'
+require_relative 'op/control'
+require_relative 'op/list'
+require_relative 'op/io'
+require_relative 'op/call'
+require_relative 'op/loop'
+require_relative 'op/element'
+require_relative 'op/regexp'
+require_relative 'op/builtin'
+
+module Peruby
+  # Converts immutable syntax nodes into executable operation trees.
+  class Compiler
+    def initialize(predeclared: nil)
+      @predeclared = predeclared
+    end
+
+    def compile(node)
+      name = node.class.name.split('::').last.gsub(/([A-Z]+)([A-Z][a-z])/, '\\1_\\2')
+                 .gsub(/([a-z\d])([A-Z])/, '\\1_\\2').downcase
+      method = "compile_#{name}"
+      raise CompileError, "Unsupported syntax node #{node.class}" unless respond_to?(method, true)
+
+      send(method, node)
+    end
+
+    private
+
+    def compile_program(node) = Op::Sequence.new(node.statements.map { |item| compile(item) })
+    def compile_block(node) = Op::Sequence.new(node.statements.map { |item| compile(item) }, scoped: true)
+    def compile_bare_block(node) = Op::BareBlock.new(compile(node.body), node.label)
+    def compile_nop(_node) = Op::Literal.new(nil)
+    def compile_literal(node) = Op::Literal.new(node.value)
+    def compile_bareword(node) = Op::Literal.new(node.value)
+    def compile_variable(node) = Op::Variable.new(node.sigil, node.name)
+
+    def compile_dereference(node)
+      operation = raw_dereference(node)
+      node.kind == :code ? Op::CodeCall.new(operation, Op::List.new([]), inherit: true) : operation
+    end
+
+    def compile_dereference_expr(node) = Op::DereferenceExpression.new(node.kind, compile(node.expression))
+    def compile_group(node) = Op::Group.new(compile(node.expression))
+
+    def compile_unary(node)
+      expression = if node.operator == :reference && node.expression.is_a?(Node::Dereference)
+                     raw_dereference(node.expression)
+                   else
+                     compile(node.expression)
+                   end
+      Op::Unary.new(node.operator, expression)
+    end
+
+    def compile_binary(node) = Op::Binary.new(node.operator, compile(node.left), compile(node.right))
+
+    def compile_ternary(node)
+      Op::Ternary.new(compile(node.condition), compile(node.true_expression), compile(node.false_expression))
+    end
+
+    def compile_assign(node) = Op::Assign.new(node.operator, compile(node.left), compile(node.right))
+    def compile_list(node) = Op::List.new(node.items.map { |item| compile(item) })
+    def compile_array_literal(node) = Op::ArrayLiteral.new(node.items.map { |item| compile(item) })
+    def compile_hash_literal(node) = Op::HashLiteral.new(node.items.map { |item| compile(item) })
+    def compile_match(node) = Op::Match.new(node.quote)
+    def compile_regexp(node) = Op::RegexpLiteral.new(node.quote)
+    def compile_bind(node) = Op::Bind.new(compile(node.subject), compile(node.pattern), node.negated)
+    def compile_substitute(node) = Op::Substitute.new(node.quote)
+    def compile_transliterate(node) = Op::Transliterate.new(node.quote)
+    def compile_readline(node) = Op::Readline.new(node.handle)
+    def compile_file_test(node) = Op::FileTest.new(node.operator, compile(node.expression))
+    def compile_postfix(node) = Op::Postfix.new(node.operator, compile(node.expression))
+    def compile_prefix(node) = Op::Prefix.new(node.operator, compile(node.expression))
+    def compile_modifier_for(node) = Op::ModifierFor.new(compile(node.expression), compile(node.list))
+    def compile_loop_jump(node) = Op::LoopJump.new(node.kind, node.label)
+    def compile_package(node) = Op::PackageSwitch.new(node.name)
+    def compile_package_block(node) = Op::PackageBlock.new(node.name, compile(node.body))
+
+    def compile_modifier_condition(node)
+      condition = compile(node.condition)
+      expression = compile(node.expression)
+      return Op::Conditional.new(condition, expression, nil) if node.kind == :if
+      return Op::Conditional.new(Op::Unary.new(:not, condition), expression, nil) if node.kind == :unless
+
+      Op::While.new(condition, expression, node.kind == :until, nil, nil)
+    end
+
+    def compile_map(node) = Op::Map.new(compile(node.body), compile(node.list), node.grep)
+
+    def compile_sort(node)
+      comparator = node.comparator && compile_sort_comparator(node.comparator)
+      Op::Sort.new(comparator, compile(node.list), node.package)
+    end
+
+    def compile_use(node) = Op::Use.new(node.module_name, node.imports, node.disable, node.package)
+    def compile_require(node) = Op::Require.new(compile(node.expression))
+    def compile_eval(node) = Op::Eval.new(compile(node.body), node.string, file: node.file)
+    def compile_constant(node) = Op::ConstantDef.new(node.name, compile(node.value))
+
+    def compile_my(node)
+      initializer = node.initializer && compile(node.initializer)
+      values = @predeclared&.[](node)
+      if node.variable.is_a?(Node::Variable)
+        return Op::Declare.new(node.variable.sigil, node.variable.name, initializer, values&.first)
+      end
+
+      variables = node.variable.items.map { |item| [item.sigil, item.name] }
+      Op::DeclareList.new(variables, initializer, values)
+    end
+
+    def compile_state(node)
+      raise CompileError, 'state list declarations are not supported' unless node.variable.is_a?(Node::Variable)
+
+      value = @predeclared&.[](node)&.first
+      Op::State.new(node.variable.sigil, node.variable.name, node.initializer && compile(node.initializer), value)
+    end
+
+    def compile_our(node)
+      initializer = node.initializer && compile(node.initializer)
+      Op::GlobalDeclare.new(node.variable.sigil, node.variable.name, initializer)
+    end
+
+    def compile_local(node)
+      target = compile(node.target)
+      assignment = node.initializer && Op::Assign.new(:'=', target, compile(node.initializer))
+      Op::Local.new(target, assignment)
+    end
+
+    def compile_if(node)
+      otherwise = node.else_block && compile(node.else_block)
+      Op::Conditional.new(compile(node.condition), compile(node.then_block), otherwise)
+    end
+
+    def compile_while(node)
+      condition = compile(node.condition)
+      condition = Op::ReadlineCondition.new(condition) if node.condition.is_a?(Node::Readline)
+      continuation = node.continue_block && compile(node.continue_block)
+      Op::While.new(condition, compile(node.body), node.until_loop, continuation, node.label)
+    end
+
+    def compile_c_for(node)
+      Op::CFor.new(node.initializer && compile(node.initializer), node.condition && compile(node.condition),
+                   node.step && compile(node.step), compile(node.body),
+                   node.continue_block && compile(node.continue_block), node.label)
+    end
+
+    def compile_print(node) = Op::Print.new(node.handle, node.arguments && compile(node.arguments))
+    def compile_printf(node) = Op::Printf.new(node.handle, node.arguments && compile(node.arguments))
+    def compile_say(node) = Op::Say.new(node.handle, node.arguments && compile(node.arguments))
+    def compile_sub_def(node) = Op::SubDef.new(node.name, node.prototype, compile_sub_body(node.body))
+    def compile_anon_sub(node) = Op::AnonSub.new(node.prototype, compile_sub_body(node.body))
+
+    def compile_sub_body(block)
+      Op::Sequence.new(block.statements.map { |statement| compile(statement) })
+    end
+
+    # rubocop:disable-next Metrics/AbcSize
+    def compile_call(node)
+      return compile_split(node) if node.name == 'split'
+      return Op::RefKind.new(compile(node.arguments.first)) if node.name == 'ref'
+      if node.name == 'push'
+        return Op::Push.new(compile(node.arguments.first), node.arguments.drop(1).map { |item| compile(item) })
+      end
+      return Op::Exists.new(compile(node.arguments.first)) if node.name == 'exists'
+      return Op::Delete.new(compile(node.arguments.first)) if node.name == 'delete'
+      return Op::Bless.new(node.arguments.map { |item| compile(item) }) if node.name == 'bless'
+      return Op::Undef.new(compile(node.arguments.first)) if node.name == 'undef'
+
+      builtin_name = node.name.rpartition('::').last
+      builtin_module = node.name.rpartition('::').first
+      core_builtin = builtin_module.empty? || ModuleLoader::INTERNAL_MODULES.include?(builtin_module)
+      if core_builtin && Op::Builtin.supports?(builtin_name)
+        arguments = node.arguments.map { |item| compile(item) }
+        return Op::Builtin.new(builtin_name, arguments, file: node.file, line: node.line)
+      end
+
+      Op::Call.new(node.name, Op::List.new(node.arguments.map { |item| compile(item) }))
+    end
+
+    def compile_split(node)
+      arguments = node.arguments.map do |item|
+        item.is_a?(Node::Match) ? Op::RegexpLiteral.new(item.quote) : compile(item)
+      end
+      Op::Split.new(arguments)
+    end
+
+    def compile_code_call(node)
+      Op::CodeCall.new(compile(node.receiver), Op::List.new(node.arguments.map { |item| compile(item) }))
+    end
+
+    def compile_method_call(node)
+      name = node.name.is_a?(Node::Variable) ? compile(node.name) : node.name
+      arguments = Op::List.new(node.arguments.map { |item| compile(item) })
+      Op::MethodCall.new(compile(node.receiver), name, arguments)
+    end
+
+    def compile_return(node) = Op::Return.new(node.expression && compile(node.expression))
+    def compile_goto(node) = Op::Goto.new(node.name)
+
+    def compile_for_each(node)
+      continuation = node.continue_block && compile(node.continue_block)
+      Op::ForEach.new(node.variable.sigil, node.variable.name, compile(node.list), compile(node.body), continuation,
+                      node.label)
+    end
+
+    def compile_element(node)
+      slice = (node.container.respond_to?(:sigil) && node.container.sigil != :scalar) ||
+              ([Node::Dereference, Node::DereferenceExpr].any? { |type| node.container.is_a?(type) } &&
+                %i[array hash].include?(node.container.kind))
+      key_value = node.container.respond_to?(:sigil) && node.container.sigil == :hash
+      Op::Element.new(compile(node.container), compile(node.key), node.kind, node.dereference, slice:, key_value:)
+    end
+
+    def raw_dereference(node) = Op::Dereference.new(node.kind, node.name)
+
+    def compile_sort_comparator(comparator)
+      comparator.is_a?(String) ? comparator : compile(comparator)
+    end
+  end
+end

data/lib/peruby/errors.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Error < StandardError; end
+  class CompileError < Error; end
+
+  # Internal non-local control transfer for loop operators.
+  class LoopControl < StandardError
+    attr_reader :kind, :label
+
+    def initialize(kind, label = nil)
+      @kind = kind
+      @label = label
+      super([kind, label].compact.join(' '))
+    end
+  end
+
+  # Non-error control flow for Perl's exit builtin.
+  class PerlExit < StandardError
+    attr_reader :status
+
+    def initialize(status)
+      @status = status
+      super("exit #{status}")
+    end
+  end
+
+  # Runtime exception retaining Perl's scalar error value.
+  class PerlError < Error
+    attr_reader :value
+
+    def initialize(message = nil, value: nil, formatted: false)
+      @value = value.nil? ? message : value
+      @formatted = formatted
+      super(message)
+    end
+
+    def formatted? = @formatted
+  end
+end

data/lib/peruby/lexer/heredoc.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Mutable payload filled when the lexer reaches the line after a heredoc declaration.
+    Heredoc = Struct.new(:terminator, :body, :interpolate, :indent, keyword_init: true)
+  end
+end

data/lib/peruby/lexer/keywords.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Perl keywords grouped by their grammar role.
+    module Keywords
+      RESERVED = %w[
+        BEGIN CHECK END INIT UNITCHECK and break cmp continue CORE default do else elsif eq eval evalbytes for foreach
+        format ge given goto gt if last le local lt m my ne next no or our package q qq qr qw qx redo require return s
+        state sub tr unless until use when while xor y
+      ].to_h { |word| [word, word.upcase.to_sym] }.freeze
+
+      NAMED_UNARY = %w[
+        abs chdir chomp chop chr chroot cos defined delete each eof eval exists exp fc fileno getc gethostbyname
+        getnetbyname getpgrp getprotobyname getprotobynumber getpwnam getpwuid getgrnam getgrgid hex int keys lc lcfirst
+        length localtime
+        lock log lstat oct ord quotemeta rand readlink ref reverse rewinddir scalar sin sqrt stat study uc ucfirst umask
+        undef values write
+      ].freeze
+
+      LISTOP = %w[
+        accept alarm all any atan2 bind binmode bless caller chmod chown close closedir connect crypt dbmclose dbmopen
+        die dump exec exit fcntl first flock fork formline gethostbyaddr getnetbyaddr getpeername getpriority
+        getservbyname getservbyport getsockname getsockopt glob gmtime grep import index ioctl join kill link listen
+        map mkdir msgctl msgget msgrcv msgsnd none open opendir pack pipe pop pos print printf prototype push read
+        readdir reduce
+        readline readpipe recv rename reset rindex rmdir say seek seekdir select semctl semget semop send setpgrp
+        setpriority
+        setsockopt shift shmctl shmget shmread shmwrite shutdown sleep socket socketpair sort splice split sprintf srand
+        substr symlink syscall sysopen sysread sysseek system syswrite tell telldir tie tied time truncate unlink
+        unpack unshift untie utime vec wait waitpid wantarray warn
+      ].freeze
+
+      FUNC0 = %w[
+        caller endgrent endhostent endnetent endprotoent endpwent endservent eof getgrent gethostent getlogin getnetent
+        getprotoent getppid getpwent getservent setgrent sethostent setnetent setprotoent setpwent setservent time times
+        wantarray
+      ].freeze
+      SPECIAL_VARIABLES = ['_', '@', '!', '/', '\\', ',', '"', ';', '.', '0', '?', '$', '|', '^W', '^O', ']', '[',
+                           '+', '-', '`', "'", '&'].freeze
+
+      module_function
+
+      def type(word, expect)
+        return :CONSTANT if word == 'constant'
+        return :NOT if word == 'not'
+        return operator_word(word) if expect == :operator && operator_word(word)
+        return word.upcase.to_sym if %w[print printf say sort map grep].include?(word)
+        return RESERVED[word] if RESERVED.key?(word)
+        return :NAMED_UNARY if NAMED_UNARY.include?(word)
+        return :LISTOP if LISTOP.include?(word)
+        return :FUNC if FUNC0.include?(word)
+
+        nil
+      end
+
+      def operator_word(word)
+        { 'x' => :REPEAT, 'lt' => :SLT, 'gt' => :SGT, 'le' => :SLE, 'ge' => :SGE,
+          'eq' => :SEQ, 'ne' => :SNE, 'cmp' => :SCMP, 'and' => :AND, 'or' => :OR, 'xor' => :XOR }[word]
+      end
+    end
+  end
+end