peruby 0.1.0

data/lib/peruby/lexer/number.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Converts a Perl numeric literal to a Ruby value.
+    module Number
+      PATTERN = /\A(?:
+        v\d+(?:\.\d+)+ | 0[xX][0-9a-fA-F_]+ | 0[bB][01_]+ | 0[oO][0-7_]+ | 0[0-7_]+ |
+        (?:\d[\d_]*\.?[\d_]*|\.[\d_]+)(?:[eE][+-]?[\d_]+)?
+      )/x
+
+      module_function
+
+      def scan(source)
+        literal = PATTERN.match(source)&.[](0)
+        literal = literal.chop if literal&.end_with?('.') && source[literal.length] == '.'
+        [literal, convert(literal)] if literal
+      end
+
+      def convert(literal)
+        clean = literal.delete('_')
+        return clean.delete_prefix('v').split('.').map(&:to_i) if clean.start_with?('v')
+        return clean.to_i(16) if clean.match?(/\A0x/i)
+        return clean.to_i(2) if clean.match?(/\A0b/i)
+        return clean.to_i(8) if clean.match?(/\A0o/i) || clean.match?(/\A0[0-7]+\z/)
+        return clean.to_f if clean.match?(/[.e]/i)
+
+        clean.to_i
+      end
+    end
+  end
+end

data/lib/peruby/lexer/quote_like.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Raw quote-like payload. Interpolation is compiled in P7.
+    Quote = Struct.new(:operator, :parts, :modifiers, :interpolate, keyword_init: true)
+
+    # Scans q/qq/qw/qr/m/s/tr/y delimiters without interpreting their contents.
+    class QuoteLike
+      PAIRS = { '(' => ')', '[' => ']', '{' => '}', '<' => '>' }.freeze
+      TWO_PART = %i[s tr y].freeze
+
+      def initialize(source, index)
+        @source = source
+        @index = index
+      end
+
+      def scan(operator)
+        skip_space if TWO_PART.include?(operator) && paired_delimiter?
+        open = take
+        raise CompileError, "Missing delimiter for #{operator}" unless open
+
+        parts = [scan_part(open)]
+        parts << scan_second(open) if TWO_PART.include?(operator)
+        modifiers = take_while(/[a-z]/)
+        [Quote.new(operator:, parts:, modifiers:, interpolate: open != "'"), @index]
+      end
+
+      private
+
+      def scan_second(first_open)
+        skip_space if PAIRS.key?(first_open)
+        open = PAIRS.key?(first_open) ? take : first_open
+        scan_part(open)
+      end
+
+      def scan_part(open)
+        close = PAIRS.fetch(open, open)
+        depth = 1
+        output = +''
+        while (character = take)
+          return output if character == close && (depth -= 1).zero?
+
+          depth += 1 if character == open && open != close
+          output << character
+          output << take.to_s if character == '\\'
+        end
+        raise CompileError, "Can't find string terminator #{close.inspect} anywhere before EOF"
+      end
+
+      def paired_delimiter?
+        PAIRS.key?(@source[@index])
+      end
+
+      def skip_space
+        @index += 1 while @source[@index]&.match?(/\s/)
+      end
+
+      def take
+        character = @source[@index]
+        @index += 1 if character
+        character
+      end
+
+      def take_while(pattern)
+        start = @index
+        @index += 1 while @source[@index]&.match?(pattern)
+        @source[start...@index]
+      end
+    end
+  end
+end

data/lib/peruby/lexer/source_scanner.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Maintains source position and skips comments, POD, and whitespace.
+    module SourceScanner
+      private
+
+      def scan_end_marker
+        return unless @column == 1 && remaining.match?(/\A__(?:END|DATA)__\s*(?:\n|\z)/)
+
+        marker = remaining.match(/\A__(?:END|DATA)__[^\n]*(?:\n|\z)/)[0]
+        advance(marker)
+        @data = remaining
+        advance(remaining)
+        [:DATA, @data]
+      end
+
+      def skip_ignored
+        loop do
+          skipped = skip_horizontal || skip_comment || skip_pod || skip_newline
+          break unless skipped
+        end
+      end
+
+      def skip_horizontal
+        text = remaining.match(/\A[^\S\n]+/)&.[](0)
+        advance(text) if text
+      end
+
+      def skip_comment
+        return unless current == '#'
+
+        advance(remaining.match(/\A[^\n]*/)[0])
+      end
+
+      def skip_pod
+        return unless @column == 1 && @state.expect == :state && remaining.match?(/\A=\w/)
+
+        finish = remaining.match(/^=cut\s*$\n?/)&.end(0)
+        raise CompileError, location_message('POD document has no =cut') unless finish
+
+        advance(remaining[0...finish])
+      end
+
+      def skip_newline
+        return unless current == "\n"
+
+        advance("\n")
+        consume_heredocs unless @pending_heredocs.empty?
+        true
+      end
+
+      def take_match(pattern)
+        match = pattern.match(remaining)&.[](0)
+        advance(match) if match
+      end
+
+      def rewind(index, column)
+        @index = index
+        @column = column
+        nil
+      end
+
+      def term_starts?(character)
+        character&.match?(/["'$@%*(&\d]/)
+      end
+
+      def delimiter?(character)
+        character && !character.match?(/[A-Za-z0-9_]/)
+      end
+
+      def illegal_character
+        raise CompileError, location_message("Unrecognized character #{current.inspect}")
+      end
+
+      def location_message(message)
+        "#{message} at #{@file} line #{@line}."
+      end
+
+      def advance(text)
+        return unless text
+
+        @index += text.length
+        lines = text.count("\n")
+        @line += lines
+        @column = lines.zero? ? @column + text.length : text.length - text.rindex("\n")
+        text
+      end
+
+      def remaining
+        @source[@index..]
+      end
+
+      def current
+        @source[@index]
+      end
+
+      def end?
+        @index >= @source.length
+      end
+    end
+  end
+end

data/lib/peruby/lexer/state.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Tracks whether the lexer expects a term, operator, statement, block, or reference.
+    class State
+      STATES = %i[state term operator block ref].freeze
+      Frame = Data.define(:open, :kind, :line)
+
+      attr_reader :brackets
+      attr_accessor :expect
+
+      def initialize
+        @expect = :state
+        @brackets = []
+      end
+
+      def push(open, kind, line)
+        @brackets << Frame.new(open, kind, line)
+      end
+
+      def pop(close)
+        frame = @brackets.pop
+        raise CompileError, "Unmatched #{close}" unless frame
+
+        frame
+      end
+
+      def term!
+        @expect = :operator
+      end
+
+      def operator!
+        @expect = :term
+      end
+
+      def statement!
+        @expect = :state
+      end
+
+      def block!
+        @expect = :block
+      end
+    end
+  end
+end

data/lib/peruby/lexer/structure_scanner.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Scans operators, brackets, and heredoc declarations and bodies.
+    module StructureScanner
+      private
+
+      def scan_bracket
+        character = current
+        return open_bracket(character) if '({['.include?(character)
+        return close_bracket(character) if ')}]'.include?(character)
+
+        nil
+      end
+
+      def open_bracket(character)
+        @sort_comparator_pending = false if @sort_comparator_pending && character == '{'
+        kind, type = bracket_kind(character)
+        advance(character)
+        @state.push(character, kind, @line)
+        kind == :block ? @state.statement! : @state.operator!
+        [type, character]
+      end
+
+      def bracket_kind(character)
+        return anonymous_sub_block if @sub_name_pending && character == '{'
+
+        return arrow_bracket(character) if @after_arrow
+
+        return [@state.expect == :operator ? :subscript : :anon_array, :'['] if character == '['
+
+        if character == '('
+          kind = @condition_pending ? :condition : :paren
+          @condition_pending = false
+          return [kind, :'(']
+        end
+        return %i[deref_block {] if @indirect_print
+        return %i[subscript SUBSCRIPT_LBRACE] if @state.expect == :operator
+        return %i[block {] if @state.expect == :block
+
+        hash = intuit_hash?
+        [hash ? :anon_hash : :block, hash ? :HASHREF_LBRACE : :'{']
+      end
+
+      def anonymous_sub_block
+        @sub_name_pending = false
+        %i[anon_sub {]
+      end
+
+      def arrow_bracket(character)
+        @after_arrow = false
+        return [:subscript, :'['] if character == '['
+        return %i[subscript SUBSCRIPT_LBRACE] if character == '{'
+
+        [:paren, :'(']
+      end
+
+      def close_bracket(character)
+        frame = @state.pop(character)
+        advance(character)
+        if frame.kind == :block
+          @state.statement!
+        elsif frame.kind == :condition
+          @state.block!
+        else
+          @state.term!
+        end
+        [character.to_sym, character]
+      end
+
+      def intuit_hash?
+        body = remaining[1..]
+        body.match?(/\A\s*}/) || body.match?(/\A\s*(?:[\w$]+|'[^']*'|"[^"]*"|\d+)\s*(?:=>|,|})/)
+      end
+
+      def scan_operator
+        operator = SORTED_OPERATORS.find { |candidate| remaining.start_with?(candidate) }
+        return unless operator
+        return if operator == '/' && @state.expect != :operator
+
+        advance(operator)
+        type = OPERATORS.fetch(operator)
+        @after_arrow = true if type == :ARROW
+        @indirect_print = false if type == :';'
+        type == :';' ? @state.statement! : operator_term_transition(type)
+        [type, operator]
+      end
+
+      def operator_term_transition(type)
+        %i[INC DEC].include?(type) ? @state.term! : @state.operator!
+      end
+
+      def operator_type?(type)
+        %i[REPEAT SLT SGT SLE SGE SEQ SNE SCMP AND OR XOR].include?(type)
+      end
+
+      def scan_heredoc
+        return unless @state.expect != :operator && remaining.start_with?('<<')
+
+        match = remaining.match(/\A<<(~?)(?:(["'])([^"']+)\2|\\?([A-Za-z_]\w*))/)
+        return unless match
+
+        advance(match[0])
+        heredoc = Heredoc.new(terminator: match[3] || match[4], interpolate: heredoc_interpolates?(match),
+                              indent: !match[1].empty?)
+        @pending_heredocs << heredoc
+        @state.term!
+        [:HEREDOC, heredoc]
+      end
+
+      def heredoc_interpolates?(match)
+        match[2] != "'" && !match[0].start_with?('<<\\')
+      end
+
+      def consume_heredocs
+        @pending_heredocs.each { |heredoc| consume_heredoc(heredoc) }
+        @pending_heredocs.clear
+      end
+
+      def consume_heredoc(heredoc)
+        lines = []
+        loop do
+          missing_heredoc!(heredoc) if end?
+          line = remaining.match(/\A.*(?:\n|\z)/)[0]
+          advance(line)
+          break if heredoc_end?(line.chomp, heredoc)
+
+          lines << line
+        end
+        heredoc.body = heredoc.indent ? dedent(lines) : lines.join
+      end
+
+      def missing_heredoc!(heredoc)
+        message = "Can't find string terminator #{heredoc.terminator.inspect} anywhere before EOF"
+        raise CompileError, location_message(message)
+      end
+
+      def heredoc_end?(line, heredoc)
+        heredoc.indent ? line.lstrip == heredoc.terminator : line == heredoc.terminator
+      end
+
+      def dedent(lines)
+        width = lines.reject { |line| line.strip.empty? }.map { |line| line[/\A\s*/].length }.min || 0
+        lines.map { |line| line.sub(/\A[ \t]{0,#{width}}/, '') }.join
+      end
+    end
+  end
+end

data/lib/peruby/lexer/term_scanner.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+module Peruby
+  class Lexer
+    # Scans literals, variables, words, and quote-like terms.
+    module TermScanner # rubocop:disable Metrics/ModuleLength
+      private
+
+      def scan_number
+        scanned = Number.scan(remaining)
+        return unless scanned
+
+        literal, value = scanned
+        advance(literal)
+        @module_name_pending = false
+        @state.term!
+        [literal.start_with?('v') ? :VERSION : :NUMBER, value]
+      end
+
+      # rubocop:disable-next Metrics/AbcSize
+      def scan_variable
+        sigil = current
+        return unless SIGILS.key?(sigil)
+        return if @state.expect == :operator && sigil != '$'
+
+        start = @index
+        column = @column
+        advance(sigil)
+        return scan_braced_dereference(sigil, start) if braced_dereference?
+        return scan_braced_variable(sigil, start) if current == '{'
+        return finish_variable(sigil, start, advance('$') && '$') if sigil == '$' && current == '$' &&
+                                                                     !@source[@index + 1].to_s.match?(/[\w$]/)
+
+        advance('$') while current == '$'
+        advance('#') if sigil == '$' && current == '#'
+        name = scan_variable_name
+        return rewind(start, column) unless name
+
+        finish_variable(sigil, start, name)
+      end
+
+      def finish_variable(sigil, start, name)
+        @after_arrow = false
+        @state.term!
+        type = variable_type(sigil, @source[start...@index])
+        type = :FILEHANDLE if @indirect_print && term_starts?(remaining.lstrip[0])
+        @indirect_print = false
+        [type, name]
+      end
+
+      def scan_braced_variable(sigil, start)
+        advance('{')
+        name = take_match(/\A(?:\^[A-Z_]+|[A-Za-z_]\w*(?:::\w+)*)/)
+        raise CompileError, location_message('Bad name after sigil') unless name && current == '}'
+
+        advance('}')
+        @state.term!
+        [variable_type(sigil, @source[start...@index]), name]
+      end
+
+      def scan_braced_dereference(sigil, _start)
+        advance('{')
+        @state.push('{', :subscript, @line)
+        @state.operator!
+        [{ '$' => :SCALAR_DEREF_OPEN, '@' => :ARRAY_DEREF_OPEN,
+           '%' => :HASH_DEREF_OPEN, '&' => :CODE_DEREF_OPEN }.fetch(sigil), sigil]
+      end
+
+      def braced_dereference?
+        return false unless current == '{'
+
+        next_character = @source[(@index + 1)..].lstrip[0]
+        SIGILS.key?(@source[@index + 1]) || (next_character && '{[('.include?(next_character))
+      end
+
+      def scan_variable_name
+        take_match(/\A(?:\^[A-Z_]+|\d+|[A-Za-z_]\w*(?:::\w+)*|[^\w\s])/)
+      end
+
+      def variable_type(sigil, literal)
+        return :ARRAY_LAST if literal.start_with?('$#')
+        return :SPECIAL_VAR if literal == '$$' || literal.match?(%r{\A\$[\d_@!/\\,";.?|^`'&+-]})
+        return :SCALAR_DEREF if literal.start_with?('$$')
+        return dereference_type(sigil) if literal[1] == '$'
+
+        SIGILS.fetch(sigil)
+      end
+
+      def dereference_type(sigil)
+        { '$' => :SCALAR_DEREF, '@' => :ARRAY_DEREF, '%' => :HASH_DEREF, '&' => :CODE_DEREF }.fetch(sigil)
+      end
+
+      def scan_word # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        word = remaining.match(/\A[A-Za-z_]\w*(?:::\w+)*(?:::)?/)&.[](0)
+        return unless word
+        return scan_quote_operator(word) if quote_operator?(word) && !@sub_name_pending && !@after_arrow
+
+        advance(word)
+        sub_name = @sub_name_pending
+        package_name = @package_name_pending
+        type = if package_name
+                 @package_name_pending = false
+                 :PACKAGE_NAME
+               elsif @module_name_pending && word != 'constant'
+                 @module_name_pending = false
+                 :PACKAGE_NAME
+               else
+                 @module_name_pending = false if word == 'constant'
+                 sub_name ? :SUB_NAME : word_type(word)
+               end
+        @module_name_pending = true if %i[USE NO].include?(type)
+        @package_name_pending = true if type == :PACKAGE
+        expression = scan_map_expression(type)
+        return expression if expression
+
+        @sub_name_pending = false
+        @prototype_pending = sub_name && remaining.lstrip.start_with?('(')
+        if @sort_comparator_pending
+          type = :SORT_SUB if type == :FUNC
+          @sort_comparator_pending = false
+        end
+        @after_arrow = false
+        transition_word(type)
+        @state.block! if package_name && remaining.lstrip.start_with?('{')
+        [type, word]
+      end
+
+      def scan_map_expression(type)
+        return unless %i[MAP GREP].include?(type) && !remaining.lstrip.start_with?('{')
+
+        comma = top_level_comma(remaining)
+        return unless comma
+
+        source = remaining[0...comma].strip
+        advance(remaining[0..comma])
+        @indirect_print = false
+        @state.operator!
+        [type == :MAP ? :MAP_EXPR : :GREP_EXPR, source]
+      end
+
+      def top_level_comma(source)
+        brackets = []
+        quote = nil
+        escaped = false
+        source.each_char.with_index do |character, index|
+          if escaped
+            escaped = false
+          elsif character == '\\'
+            escaped = true
+          elsif quote
+            quote = nil if character == quote
+          elsif %w[' "].include?(character)
+            quote = character
+          elsif '([{'.include?(character)
+            brackets << character
+          elsif ')]}'.include?(character)
+            brackets.pop
+          elsif character == ',' && brackets.empty?
+            return index
+          end
+        end
+        nil
+      end
+
+      def scan_prototype
+        anonymous = @sub_name_pending && current == '('
+        return unless @prototype_pending || anonymous
+
+        prototype = remaining.match(/\A\(([^)]*)\)/)
+        raise CompileError, location_message('Malformed prototype') unless prototype
+
+        value = prototype[1].gsub(/\s+/, '')
+        raise CompileError, location_message('Illegal character in prototype') unless value.match?(/\A[$@%&*+_;\\]*\z/)
+
+        advance(prototype[0])
+        @prototype_pending = false
+        @sub_name_pending = false if anonymous
+        @state.block!
+        [:PROTOTYPE, value]
+      end
+
+      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def word_type(word)
+        return word.upcase.to_sym if %w[print printf say].include?(word)
+        return :BAREWORD if @after_arrow
+
+        callable = Keywords::NAMED_UNARY.include?(word) || Keywords::LISTOP.include?(word)
+        return :FUNC if callable && remaining.lstrip.start_with?('(')
+
+        keyword = Keywords.type(word, @state.expect)
+        return keyword if keyword
+
+        if @known_subs.key?(word)
+          return :LISTOP if @known_subs[word] == :listop && !remaining.lstrip.start_with?('(')
+
+          return :FUNC
+        end
+        return :PACKAGE_NAME if remaining.lstrip.start_with?('->')
+        return :FILEHANDLE if @indirect_print && word.match?(/\A[A-Z][A-Z0-9_]*\z/)
+        return :STRING if remaining.lstrip.start_with?('=>')
+        return :FUNC if word.include?('::') && remaining.lstrip.start_with?('(')
+        return :PACKAGE_NAME if word.include?('::')
+        return :FUNC if remaining.lstrip.start_with?('(')
+
+        :BAREWORD
+      end
+
+      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def transition_word(type)
+        @indirect_print = false if type == :FILEHANDLE
+        @indirect_print = false if type == :FUNC
+        @indirect_print = false if type == :LISTOP
+        if type == :SUB
+          @sub_name_pending = true
+          return @state.block!
+        end
+        return @state.block! if %i[SUB_NAME ELSE DO BEGIN CHECK INIT END CONTINUE].include?(type)
+        return remaining.lstrip.start_with?('{') ? @state.block! : @state.operator! if type == :EVAL
+
+        if %i[MAP GREP SORT].include?(type)
+          @indirect_print = false
+          block = remaining.lstrip.start_with?('{')
+          @sort_comparator_pending = true if type == :SORT && !block
+          return block ? @state.block! : @state.operator!
+        end
+
+        @condition_pending = true if %i[IF ELSIF UNLESS WHILE UNTIL FOR FOREACH].include?(type)
+        operators = %i[IF UNLESS WHILE UNTIL FOR FOREACH RETURN MY OUR LOCAL STATE PRINT PRINTF SAY LISTOP
+                       SORT_SUB NAMED_UNARY NOT GOTO]
+        if operators.include?(type) || operator_type?(type)
+          @indirect_print = true if %i[PRINT PRINTF SAY].include?(type)
+          @state.operator!
+        else
+          @state.term!
+        end
+      end
+
+      def scan_quote
+        return scan_simple_quote if %w[' " `].include?(current)
+        return scan_regex if current == '/' && @state.expect != :operator
+
+        nil
+      end
+
+      def scan_simple_quote
+        delimiter = current
+        quote, finish = QuoteLike.new(@source, @index).scan(delimiter == '`' ? :backtick : :q)
+        advance(@source[@index...finish])
+        quote.interpolate = delimiter != "'"
+        @state.term!
+        [simple_quote_type(delimiter, quote), quote]
+      end
+
+      def simple_quote_type(delimiter, quote)
+        return :BACKTICK if delimiter == '`'
+
+        quote.interpolate ? :ISTRING : :STRING
+      end
+
+      def scan_regex
+        quote, finish = QuoteLike.new(@source, @index).scan(:m)
+        advance(@source[@index...finish])
+        @state.term!
+        [:MATCH, quote]
+      end
+
+      def quote_operator?(word)
+        QUOTE_OPERATORS.include?(word) && delimiter?(@source[@index + word.length])
+      end
+
+      def scan_quote_operator(word)
+        advance(word)
+        operator = word.to_sym
+        quote, finish = QuoteLike.new(@source, @index).scan(operator)
+        advance(@source[@index...finish])
+        @state.term!
+        types = { q: :STRING, qq: :ISTRING, qw: :QW, qr: :QR, m: :MATCH, s: :SUBST, tr: :TRANS, y: :TRANS }
+        [types.fetch(operator), quote]
+      end
+
+      def scan_filetest
+        return unless @state.expect != :operator && (literal = FILETEST.match(remaining)&.[](0))
+
+        advance(literal)
+        @state.operator!
+        [:FILETEST, literal]
+      end
+
+      def scan_readline
+        return if @state.expect == :operator
+
+        literal = remaining.match(/\A<(?:\$?[A-Za-z_]\w*(?:::\w+)*)?>/)&.[](0)
+        return unless literal
+
+        advance(literal)
+        @state.term!
+        [:READLINE, literal[1...-1]]
+      end
+    end
+  end
+end