peruby 0.1.0

data/lib/peruby/lexer/token.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # One lexical token and its source location.
+    Token = Data.define(:type, :value, :file, :line, :column) do
+      def to_a
+        [type, value]
+      end
+
+      def to_s
+        "#{file}:#{line}:#{column} #{type} #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/peruby/lexer.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require_relative 'lexer/token'
+require_relative 'lexer/state'
+require_relative 'lexer/keywords'
+require_relative 'lexer/number'
+require_relative 'lexer/quote_like'
+require_relative 'lexer/heredoc'
+require_relative 'lexer/term_scanner'
+require_relative 'lexer/structure_scanner'
+require_relative 'lexer/source_scanner'
+
+module Peruby
+  # Stateful Perl lexer. It resolves term/operator ambiguities without parser feedback.
+  class Lexer
+    OPERATORS = {
+      '**=' => :ASSIGNOP, '//=' => :ASSIGNOP, '&&=' => :ASSIGNOP, '||=' => :ASSIGNOP,
+      '<<=' => :ASSIGNOP, '>>=' => :ASSIGNOP, '&=' => :ASSIGNOP, '|=' => :ASSIGNOP, '^=' => :ASSIGNOP,
+      '...' => :DOTDOTDOT, '**' => :POW,
+      '//' => :DORDOR, '&&' => :ANDAND, '||' => :OROR, '=~' => :MATCHOP,
+      '!~' => :NOTMATCHOP, '==' => :EQ, '!=' => :NE, '<=' => :LE, '>=' => :GE,
+      '<=>' => :CMP, '->' => :ARROW, '=>' => :FATCOMMA, '..' => :DOTDOT,
+      '<<' => :LSHIFT, '>>' => :RSHIFT, '++' => :INC, '--' => :DEC,
+      '+=' => :ASSIGNOP, '-=' => :ASSIGNOP, '*=' => :ASSIGNOP, '/=' => :ASSIGNOP,
+      '%=' => :ASSIGNOP, '.=' => :ASSIGNOP, 'x=' => :ASSIGNOP,
+      '=' => :'=', '+' => :+, '-' => :-, '*' => :*, '/' => :/, '%' => :%, '.' => :'.',
+      '<' => :<, '>' => :>, '!' => :!, '~' => :~, '&' => :&, '|' => :|, '^' => :^,
+      '?' => :'?', ':' => :':', ',' => :',', ';' => :';', '\\' => :REFGEN
+    }.freeze
+    SORTED_OPERATORS = OPERATORS.keys.sort_by(&:length).reverse.freeze
+    SIGILS = { '$' => :SCALAR_VAR, '@' => :ARRAY_VAR, '%' => :HASH_VAR,
+               '&' => :CODE_VAR, '*' => :GLOB_VAR }.freeze
+    QUOTE_OPERATORS = %w[q qq qw qr m s tr y].freeze
+    FILETEST = /\A-[erwxofdszlpSbcugktTBAMC](?!\w)/
+
+    include TermScanner
+    include StructureScanner
+    include SourceScanner
+
+    attr_reader :state, :data
+
+    def initialize(source, file: '-e', known_subs: {}, implicit_semicolon: false)
+      @source = source
+      @file = file
+      @known_subs = known_subs
+      @implicit_semicolon_enabled = implicit_semicolon
+      @index = 0
+      @line = 1
+      @column = 1
+      @state = State.new
+      @pending_heredocs = []
+      @indirect_print = false
+      @data = nil
+    end
+
+    def tokens
+      each.to_a
+    end
+
+    def each
+      return enum_for(__method__) unless block_given?
+
+      while (token = next_token)
+        yield token
+      end
+    end
+
+    def next_token
+      skip_ignored
+      return eof if end?
+      return implicit_block_semicolon if implicit_block_semicolon?
+
+      line = @line
+      column = @column
+      type, value = scan_token
+      @last_token_type = type
+      Token.new(type, value, @file, line, column)
+    end
+
+    private
+
+    def implicit_block_semicolon?
+      @implicit_semicolon_enabled && current == '}' && %i[block anon_sub].include?(@state.brackets.last&.kind) &&
+        @implicit_block_index != @index &&
+        !%i[; {].include?(@last_token_type)
+    end
+
+    def implicit_block_semicolon
+      @implicit_block_index = @index
+      @last_token_type = :';'
+      @indirect_print = false
+      @state.statement!
+      Token.new(:';', ';', @file, @line, @column)
+    end
+
+    def scan_token
+      scanners.each do |scanner|
+        token = send(scanner)
+        return token if token
+      end
+      illegal_character
+    end
+
+    def scanners
+      %i[scan_end_marker scan_prototype scan_heredoc scan_readline scan_quote scan_number scan_variable scan_filetest
+         scan_operator scan_word scan_bracket]
+    end
+
+    def eof
+      if @state.brackets.empty?
+        return nil unless @implicit_semicolon_enabled
+        return nil if @implicit_semicolon || @last_token_type.nil? || @last_token_type == :';'
+        return nil if @last_token_type == :'}' && @state.expect != :operator
+
+        @implicit_semicolon = true
+        return Token.new(:';', ';', @file, @line, @column)
+      end
+
+      frame = @state.brackets.last
+      raise CompileError, "Unmatched #{frame.open} at #{@file} line #{frame.line}."
+    end
+  end
+end

data/lib/peruby/node.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Peruby
+  # Syntax tree node types.
+  module Node
+    # Recursive S-expression conversion shared by all immutable nodes.
+    module SExpression
+      def to_sexp
+        name = self.class.name.split('::').last.gsub(/([A-Z]+)([A-Z][a-z])/, '\\1_\\2')
+                   .gsub(/([a-z\d])([A-Z])/, '\\1_\\2').downcase.to_sym
+        values = members.reject { |member| %i[file line].include?(member) }.map { |member| sexp(public_send(member)) }
+        [name, *values]
+      end
+
+      private
+
+      def sexp(value)
+        return value.to_sexp if value.respond_to?(:to_sexp)
+        return value.map { |item| sexp(item) } if value.is_a?(Array)
+
+        value
+      end
+    end
+
+    def self.define(name, *members)
+      const_set(name, Data.define(*members, :file, :line) { include SExpression })
+    end
+
+    define :Program, :statements
+    define :Nop
+    define :Literal, :value
+    define :Bareword, :value
+    define :Variable, :sigil, :name
+    define :Unary, :operator, :expression
+    define :Binary, :operator, :left, :right
+    define :Ternary, :condition, :true_expression, :false_expression
+    define :Assign, :operator, :left, :right
+    define :List, :items
+    define :Block, :statements
+    define :BareBlock, :body, :label
+    define :If, :condition, :then_block, :else_block
+    define :While, :condition, :body, :until_loop, :continue_block, :label
+    define :CFor, :initializer, :condition, :step, :body, :continue_block, :label
+    define :My, :variable, :initializer
+    define :State, :variable, :initializer
+    define :Return, :expression
+    define :SubDef, :name, :prototype, :body
+    define :AnonSub, :prototype, :body
+    define :ForEach, :variable, :list, :body, :continue_block, :label, :declaration
+    define :LoopJump, :kind, :label
+    define :ModifierCondition, :expression, :condition, :kind
+    define :Package, :name
+    define :PackageBlock, :name, :body
+    define :Print, :handle, :arguments
+    define :Call, :name, :arguments
+    define :CodeCall, :receiver, :arguments
+    define :Element, :container, :key, :kind, :dereference
+    define :MethodCall, :receiver, :name, :arguments
+    define :ArrayLiteral, :items
+    define :HashLiteral, :items
+    define :Match, :quote
+    define :Bind, :subject, :pattern, :negated
+    define :Regexp, :quote
+    define :Substitute, :quote
+    define :Transliterate, :quote
+    define :Dereference, :kind, :name
+    define :DereferenceExpr, :kind, :expression
+    define :Group, :expression
+    define :Our, :variable, :initializer
+    define :Local, :target, :initializer
+    define :Goto, :name
+    define :Readline, :handle
+    define :FileTest, :operator, :expression
+    define :Printf, :handle, :arguments
+    define :Say, :handle, :arguments
+    define :Postfix, :operator, :expression
+    define :Prefix, :operator, :expression
+    define :ModifierFor, :expression, :list
+    define :Map, :body, :list, :grep
+    define :Sort, :comparator, :list, :package
+    define :Use, :module_name, :imports, :disable, :package
+    define :Require, :expression
+    define :Eval, :body, :string
+    define :Constant, :name, :value
+
+    Bareword.define_method(:to_sexp) { [:literal, value] }
+  end
+end

data/lib/peruby/op/assign.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Peruby
+  module Op
+    # Scalar plain or compound assignment.
+    class Assign < Base
+      def initialize(operator, left, right)
+        @operator = operator
+        @left = left
+        @right = right
+      end
+
+      def run(env, context)
+        return assign_aggregate(env, context) if @operator == :'=' && @left.respond_to?(:aggregate_assignment?) &&
+                                                 @left.aggregate_assignment?
+
+        target = @left.lvalue(env, :list)
+        return assign_list(target, env, context) if target.is_a?(Array)
+
+        return logical_assignment(target, env, context) if logical_assignment?
+
+        value = @operator == :'=' ? @right.run(env, :scalar) : assigned_value(target.get, @right.run(env, :scalar))
+        target.set(value)
+        context == :list ? [target] : value
+      end
+
+      def to_callable
+        return super if logical_assignment?
+
+        right = @right.to_callable
+        scalar_right = @right.scalar_callable if @right.respond_to?(:scalar_callable)
+        left = @left.respond_to?(:lvalue_callable) ? @left.lvalue_callable : ->(env) { @left.lvalue(env, :list) }
+        lambda do |env, context|
+          if @operator == :'=' && @left.respond_to?(:aggregate_assignment?) && @left.aggregate_assignment?
+            next assign_aggregate(env, context, right.call(env, :list))
+          end
+
+          target = left.call(env)
+          next assign_list(target, env, context) if target.is_a?(Array)
+
+          value = if @operator == :'='
+                    scalar_right ? scalar_right.call(env) : right.call(env, :scalar)
+                  else
+                    compiled_value(target.value, scalar_right ? scalar_right.call(env) : right.call(env, :scalar))
+                  end
+          target.set(value)
+          context == :list ? [target] : value
+        end
+      end
+
+      def to_void_callable
+        return ->(env) { run(env, :void) } if logical_assignment?
+
+        right = @right.to_callable
+        scalar_right = @right.scalar_callable if @right.respond_to?(:scalar_callable)
+        left = @left.respond_to?(:lvalue_callable) ? @left.lvalue_callable : ->(env) { @left.lvalue(env, :list) }
+        lambda do |env|
+          if @operator == :'=' && @left.respond_to?(:aggregate_assignment?) && @left.aggregate_assignment?
+            next assign_aggregate(env, :void, right.call(env, :list))
+          end
+
+          target = left.call(env)
+          next assign_list(target, env, :void) if target.is_a?(Array)
+
+          right_value = scalar_right ? scalar_right.call(env) : right.call(env, :scalar)
+          value = @operator == :'=' ? right_value : compiled_value(target.value, right_value)
+          target.set(value)
+          value
+        end
+      end
+
+      private
+
+      def compiled_value(left, right)
+        if left.is_a?(Numeric) && right.is_a?(Numeric)
+          return Conv.norm_int(left + right) if @operator == :'+='
+          return Conv.norm_int(left - right) if @operator == :'-='
+          return Conv.norm_int(left * right) if @operator == :'*='
+          return Conv.divide(left, right) if @operator == :'/='
+          return Conv.modulo(left, right) if @operator == :'%='
+        end
+        assigned_value(left, right)
+      end
+
+      def assign_aggregate(env, context, cells = @right.run(env, :list))
+        values = cells.map(&:get)
+        @left.assign_values(env, values)
+        context == :list ? values.map { |value| Scalar.new(value) } : values.length
+      end
+
+      def assign_list(targets, env, context)
+        raise PerlError, 'Modification of a read-only value attempted' unless @operator == :'='
+
+        values = @right.run(env, :list).map(&:get)
+        targets.each_with_index { |target, index| target.set(values[index]) }
+        context == :list ? targets : values.length
+      end
+
+      def assigned_value(left, right) # rubocop:disable Metrics/AbcSize
+        case @operator.to_s
+        when '+=' then Conv.norm_int(Conv.to_num(left) + Conv.to_num(right))
+        when '-=' then Conv.norm_int(Conv.to_num(left) - Conv.to_num(right))
+        when '*=' then Conv.norm_int(Conv.to_num(left) * Conv.to_num(right))
+        when '/=' then Conv.divide(left, right)
+        when '%=' then Conv.modulo(left, right)
+        when '.=' then Conv.to_str(left) + Conv.to_str(right)
+        when 'x=' then Conv.repeat(left, right)
+        when '**=' then Conv.norm_int(Conv.to_num(left)**Conv.to_num(right))
+        when '<<=' then Conv.to_num(left).to_i << Conv.to_num(right).to_i
+        when '>>=' then Conv.to_num(left).to_i >> Conv.to_num(right).to_i
+        when '&=' then Conv.bitwise(:&, left, right)
+        when '|=' then Conv.bitwise(:|, left, right)
+        when '^=' then Conv.bitwise(:^, left, right)
+        end
+      end
+
+      def logical_assignment?(operator = @operator) = %i[&&= ||= //=].include?(operator)
+
+      def logical_assignment(target, env, context)
+        current = target.get
+        assign = (@operator == :'&&=' && Conv.truthy?(current)) || (@operator == :'||=' && !Conv.truthy?(current)) ||
+                 (@operator == :'//=' && current.nil?)
+        target.set(@right.run(env, :scalar)) if assign
+        context == :list ? [target] : target.get
+      end
+    end
+  end
+end