arpaka 1.0.0 → 2.0.0

data/lib/arpaka/{runtime → ruby/runtime}/lexer.rb.erb

@@ -5,7 +5,8 @@ class LexicalContext
   attr_accessor :begin_expression, :condition_do, :condition_line,
     :ternary_depth, :lambda_pending, :alias_context, :argument_label,
     :lex_state, :command_start, :label_pending, :singleton_class_depth, :control_depth,
-    :method_header, :method_body_start
+    :method_header, :method_body_start, :command_argument_candidate,
+    :command_argument_pending, :command_call_active, :command_call_argument
   attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack,
     :token_history, :state_history, :block_stack, :scope_stack, :parser_events,
     :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack,
@@ -24,6 +25,10 @@ class LexicalContext
     @control_depth = 0
     @method_header = false
     @method_body_start = false
+    @command_argument_candidate = false
+    @command_argument_pending = false
+    @command_call_active = false
+    @command_call_argument = false
     @lex_state = :expr_beg
     @command_start = true
     @delimiter_stack = []
@@ -185,7 +190,7 @@ end
 class Lexer
   Error = LexerError
 
-  attr_reader :context
+  attr_reader :context, :source_encoding
 
   KEYWORDS = {
     "class" => :keyword_class, "module" => :keyword_module,
@@ -220,17 +225,18 @@ class Lexer
     "=>" => :tASSOC, "->" => :tLAMBDA, "[]" => :tAREF, "[]=" => :tASET
   }.freeze
 
-  def initialize(source, filename: "(ruby)")
+  def initialize(source, filename: "(ruby)", initial_line: 1)
     unless source.is_a?(String)
       raise ArgumentError, "source must be a String"
     end
     unless [Encoding::UTF_8, Encoding::US_ASCII].include?(source.encoding) && source.valid_encoding?
       raise Error, "#{filename}:1:0: invalid UTF-8 source"
     end
+    @source_encoding = source_encoding_for(source)
     @source = source.b
     @filename = filename
     @index = 0
-    @line = 1
+    @line = initial_line
     @column = 0
     @previous_previous = nil
     @previous = nil
@@ -239,9 +245,30 @@ class Lexer
     @pending = []
     @heredoc_queue = []
     @deferred_suffix = nil
+    @deferred_suffix_line = nil
     @heredoc_newline_pending = false
     @class_superclass = false
     @local_variables = {}
+    @source_local_variables = source_local_variable_names
+  end
+
+  def source_encoding_for(source)
+    lines = source.lines.first(2)
+    lines.each_with_index do |line, index|
+      next if index == 1 && !lines.first.start_with?("#!")
+
+      match = line.match(/\A[ \t]*#.*?(?:coding|encoding)\s*[:=]\s*([A-Za-z0-9._-]+)/i)
+      next unless match
+
+      return Encoding.find(match[1]).name
+    end
+    source.encoding.name
+  rescue ArgumentError => error
+    raise Error, "invalid source encoding: #{error.message}"
+  end
+
+  def source_string
+    String.new(encoding: @source_encoding)
   end
 
   def each
@@ -286,7 +313,8 @@ class Lexer
     elsif @pending.empty? && @deferred_suffix
       suffix = @deferred_suffix
       @deferred_suffix = nil
-      @pending.concat(heredoc_suffix_tokens(suffix))
+      @pending.concat(heredoc_suffix_tokens(suffix, @deferred_suffix_line))
+      @deferred_suffix_line = nil
       @pending.pop if @pending.last == [0, nil]
       @pending << ["\n", nil]
       @heredoc_newline_pending = false
@@ -498,17 +526,50 @@ class Lexer
   end
 
   def update_block_context(token)
+    if ["=", :tASSOC, :tOP_ASGN].include?(token)
+      @context.command_call_active = false
+      @context.command_call_argument = false
+    end
+    if token == ")" && @context.command_argument_candidate
+      @context.command_argument_candidate = false
+      @context.command_argument_pending = true
+    elsif @context.command_argument_pending &&
+        ![:keyword_do, :keyword_do_block, :keyword_do_LAMBDA].include?(token)
+      @context.command_argument_pending = false
+    elsif @context.command_argument_candidate &&
+        ![".", :tCOLON2, :tANDDOT, "(", :tLPAREN, :tLPAREN_ARG,
+          :tIDENTIFIER, :tCONSTANT, :tFID].include?(token) &&
+        !@context.delimiter_stack.include?("(")
+      @context.command_argument_candidate = false
+    end
     case token
+    when "("
+      unless @context.command_call_argument
+        @context.command_call_active = false
+      end
+    when ","
+      @context.command_call_argument = true if @context.command_call_active &&
+        @context.delimiter_stack.empty? &&
+        (@context.block_stack.empty? || @context.block_body_at_current_delimiter?)
+    when "\n", ";"
+      @context.command_argument_candidate = false
+      @context.command_argument_pending = false
+      @context.command_call_active = false
+      @context.command_call_argument = false
+    when :keyword_do, :keyword_do_block
+      @context.command_call_active = false
+      @context.command_call_argument = false
+      @context.push_block(:do_block)
+    when :keyword_do_LAMBDA
+      @context.command_call_active = false
+      @context.command_call_argument = false
+      @context.push_block(:lambda)
     when :keyword_if, :keyword_unless, :keyword_case, :keyword_begin,
       :keyword_while, :keyword_until, :keyword_for
       @context.control_depth += 1
     when :keyword_def
       @context.method_header = true
       @context.push_scope(:method)
-    when :keyword_do, :keyword_do_block
-      @context.push_block(:do_block)
-    when :keyword_do_LAMBDA
-      @context.push_block(:lambda)
     when :tLAMBDA
       @context.push_scope(:lambda)
     when :tLSHFT
@@ -516,8 +577,12 @@ class Lexer
     when :tLAMBEG
       @context.push_block(:lambda)
     when "{", :tLBRACE_ARG
+      @context.command_call_active = false
+      @context.command_call_argument = false
       @context.push_block(:brace_block)
     when "}"
+      @context.command_call_active = false
+      @context.command_call_argument = false
       @context.pop_block if [:brace_block, :lambda].include?(@context.block_stack.last)
       @context.pop_scope if @context.scope_stack.last == :lambda
     when :keyword_end
@@ -549,6 +614,25 @@ class Lexer
       advance
     end
     word = @source.byteslice(start, @index - start).force_encoding(Encoding::UTF_8)
+    if word == "__END__" && end_data_marker?(start)
+      @index = @source.bytesize
+      return nil
+    end
+    command_start = @context.command_start ||
+      ["=", :tASSOC, :tOP_ASGN].include?(@previous)
+    if command_start
+      @context.command_call_active = true
+      @context.command_call_argument = false
+    elsif word != "do" && @context.command_call_active && !@context.command_call_argument &&
+      start.positive? && [9, 32].include?(@source.getbyte(start - 1)) &&
+      [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous)
+      @context.command_call_argument = true
+    end
+    if !@context.command_start && [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) &&
+        start.positive? && [9, 32].include?(@source.getbyte(start - 1)) &&
+        nested_parenthesized_command_argument?
+      @context.command_argument_candidate = true
+    end
     if byte == 63 && word == "defined"
       advance
       word = "defined?"
@@ -579,8 +663,9 @@ class Lexer
       advance
       return [:tIDENTIFIER, (word + "=").to_sym]
     end
-    @local_variables[word.to_sym] = true if byte == 61 && byte(1) != 61 &&
-      ![".", :tCOLON2, :tANDDOT].include?(@previous)
+    @local_variables[word.to_sym] = true if @context.lambda_pending ||
+      (byte == 61 && byte(1) != 61 &&
+      ![".", :tCOLON2, :tANDDOT].include?(@previous))
     label_in_ternary = ["(", "[", :tLPAREN, :tLPAREN_ARG, :tLBRACK, ","].include?(@previous)
     if byte == 58 && byte(1) != 58 &&
         (@context.ternary_depth.zero? || label_in_ternary) &&
@@ -615,7 +700,7 @@ class Lexer
       @context.push_condition
     end
     @context.alias_context = true if token == :keyword_alias
-    [token, word.to_sym]
+    [token, word == "__LINE__" ? @line : word.to_sym]
   rescue EncodingError
     fail!("invalid UTF-8 identifier", start)
   end
@@ -624,6 +709,17 @@ class Lexer
     value && (value == 95 || value.between?(65, 90) || value.between?(97, 122) || value >= 128 || value.between?(48, 57))
   end
 
+  def nested_parenthesized_command_argument?
+    position = @index
+    position += 1 while [9, 32].include?(@source.getbyte(position))
+    if @source.getbyte(position) == 46
+      position += 1
+      position += 1 while identifier_byte?(@source.getbyte(position))
+      position += 1 while [9, 32].include?(@source.getbyte(position))
+    end
+    @source.getbyte(position) == 40
+  end
+
   def do_token
     if @context.condition_do && @context.condition?
       @context.pop_condition
@@ -636,30 +732,48 @@ class Lexer
       :keyword_do
     elsif @previous == ")" && @context.token_history.each_cons(2).any? { |left, right| left == :tUMINUS && right == :tLPAREN }
       :keyword_do_block
+    elsif @previous == ")" && @context.command_argument_pending
+      @context.command_argument_pending = false
+      :keyword_do_block
+    elsif command_argument_receiver_call?
+      :keyword_do_block
+    elsif @context.command_call_active && @context.command_call_argument &&
+        @previous == :tIDENTIFIER && [:lambda, :proc].include?(@previous_value)
+      :keyword_do
+    elsif @previous == :tIDENTIFIER && @previous_previous == "." && @context.token_history[-3] == ")"
+      :keyword_do
+    elsif @previous == ")" && @context.command_call_active && @context.command_call_argument
+      :keyword_do_block
     elsif [")", :keyword_super, :keyword_yield].include?(@previous)
       :keyword_do
     elsif @previous == :tLAMBDA
       :keyword_do_LAMBDA
+    elsif @context.command_call_active && @context.command_call_argument
+      :keyword_do_block
     else
       :keyword_do_block
     end
   end
 
   def no_argument_block?
+    return false if @context.command_call_active && @context.command_call_argument
+    return false if command_argument_receiver_call?
     return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous)
     return false if @context.argument_label == :at && [".", :tCOLON2, :tANDDOT].include?(@previous_previous)
     return false if @context.token_history == ["+", :tINTEGER, ".", :tIDENTIFIER]
     return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous)
     return false if [:tSYMBEG, :tLABEL, :tCOLON2].include?(@previous_previous)
+    return true if @previous_value == :lambda
+    return false if @previous_previous == "," && @context.delimiter_stack.empty? &&
+      [:tIDENTIFIER, :tCONSTANT, :tFID, :tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY].include?(@context.token_history[-3])
     return true if @previous_value == :lambda && @previous_previous == :tLSHFT
     return false if [:tLSHFT, :tLAMBDA].include?(@previous_previous)
     true
   end
 
-  def no_argument_brace_block?
-    return false if [".", :tCOLON2].include?(@previous_previous)
-    return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, :tAMPER, "=", "(", ",", ":", "?"].include?(@previous_previous)
-    no_argument_block?
+  def command_argument_receiver_call?
+    @context.command_call_active && @previous == :tIDENTIFIER && @previous_previous == "." &&
+      [:tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY].include?(@context.token_history[-3])
   end
 
   def number_token(start)
@@ -692,7 +806,8 @@ class Lexer
     text = @source.byteslice(start, @index - start).delete("_")
     return [:tRATIONAL, Rational(text.delete_suffix("r"))] if suffix == 114
     return [:tIMAGINARY, Complex(0, text.delete_suffix("i").to_f)] if suffix == 105
-    [float ? :tFLOAT : :tINTEGER, float ? text.to_f : text.to_i]
+    base = 8 if !float && text.start_with?("0")
+    [float ? :tFLOAT : :tINTEGER, float ? text.to_f : Integer(text, base)]
   rescue ArgumentError, ZeroDivisionError
     fail!("invalid numeric literal", start)
   end
@@ -706,7 +821,7 @@ class Lexer
     marker = byte
     advance
     advance if marker == 64 && byte == 64
-    if marker == 36 && [33, 38, 39, 42, 43, 46, 47, 58, 60, 62, 61, 63, 96, 126].include?(byte)
+    if marker == 36 && [33, 38, 39, 42, 43, 44, 46, 47, 58, 60, 62, 61, 63, 96, 126].include?(byte)
       advance
       text = @source.byteslice(start, @index - start)
       return [:tGVAR, text.to_sym]
@@ -716,6 +831,10 @@ class Lexer
       return [:tGVAR, :"$$"]
     end
     if marker == 36 && byte && byte.between?(48, 57)
+      if byte == 48
+        advance
+        return [:tGVAR, :"$0"]
+      end
       advance while byte && byte.between?(48, 57)
       return [:tNTH_REF, @source.byteslice(start + 1, @index - start - 1).to_i]
     end
@@ -796,6 +915,27 @@ class Lexer
     fail!("invalid UTF-8 character")
   end
 
+  def source_character
+    length = utf8_character_length(byte)
+    value = @source.byteslice(@index, length).dup.force_encoding(Encoding::UTF_8)
+    advance(length)
+    value
+  end
+
+  def escaped_character
+    value = byte
+    if value >= 0x80
+      source_character
+    else
+      advance
+      value.chr.force_encoding(Encoding::UTF_8)
+    end
+  end
+
+  def append_source_character(result)
+    result << source_character
+  end
+
   def symbol_or_colon(start)
     symbol_position = @context.begin_expression || ([:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) && @context.ternary_depth.zero?)
     return operator_or_punctuation(start) unless symbol_position && byte(1) &&
@@ -811,6 +951,14 @@ class Lexer
   end
 
   def string_token(quote, start)
+    # A receiver call can be the final expression of a command-call argument:
+    # `test "description".squish do ... end`. Mark the string as an
+    # argument before consuming the receiver call so `do` binds to `test`,
+    # rather than to `squish`.
+    if @context.command_call_active && !@context.command_call_argument &&
+        @context.delimiter_stack.empty? && @context.block_stack.empty?
+      @context.command_call_argument = true
+    end
     advance
     content_tokens = read_interpolated_quoted(quote, start)
     label = byte == 58 && @context.ternary_depth.zero?
@@ -823,29 +971,34 @@ class Lexer
 
   def read_interpolated_quoted(quote, start)
     return [[:tSTRING_CONTENT, read_quoted(quote, interpolate: false, start: start)]] if quote == 39
-    content = +""
+    content = source_string
     tokens = []
     until eof?
       value = byte
       if value == quote
         advance
-        tokens << [:tSTRING_CONTENT, content] unless content.empty?
+        tokens << [:tSTRING_CONTENT, content] if tokens.empty? || !content.empty?
         return tokens
       elsif value == 92
-        content << escape_sequence(start)
+        content << escape_sequence(start, mode: :double, quote: quote)
       elsif value == 35 && byte(1) == 123
         tokens << [:tSTRING_CONTENT, content] unless content.empty?
-        content = +""
+        content = source_string
         advance(2)
+        expression_line = @line
         expression = read_interpolation_source(start)
-        inner = self.class.new(expression, filename: @filename).each.to_a
+        inner = self.class.new(expression, filename: @filename, initial_line: expression_line).each.to_a
         inner.pop if inner.last == [0, nil]
+        inner.map! { |token| token.first == "\n" ? [";", nil] : token }
         tokens << [:tSTRING_DBEG, nil]
         tokens.concat(inner)
         tokens << [:tSTRING_DEND, nil]
+      elsif value == 35 && (length = shorthand_interpolation_length)
+        tokens << [:tSTRING_CONTENT, content] unless content.empty?
+        content = source_string
+        tokens.concat(shorthand_interpolation_tokens(start, length))
       else
-        content << value.chr
-        advance
+        append_source_character(content)
       end
     end
     fail!("unterminated literal", start)
@@ -884,46 +1037,173 @@ class Lexer
   end
 
   def read_quoted(quote, interpolate:, start:)
-    result = +""
+    result = source_string
     until eof?
       value = byte
       if value == quote
         advance
         return result
       elsif value == 92
-        result << escape_sequence(start)
+        mode = quote == 39 ? :single : :double
+        result << escape_sequence(start, mode: mode, quote: quote)
       elsif value == 35 && interpolate && byte(1) == 123
         fail!("string interpolation cannot be represented by the AST", start)
       else
-        result << value
-        advance
+        append_source_character(result)
       end
     end
     fail!("unterminated literal", start)
   end
 
-  def escape_sequence(start)
+  def escape_sequence(start, mode: :double, quote: nil, delimiters: [])
     advance
     fail!("unterminated escape", start) if eof?
     value = byte
-    advance
+    escaped = escaped_character
+    case mode
+    when :single
+      return escaped if value == 92 || value == quote
+      "\\" + escaped
+    when :percent_q, :word, :regexp
+      if mode == :regexp
+        return escaped if delimiters.include?(value) && delimiters.uniq.length == 1
+        return "\\" + escaped
+      end
+      return escaped if value == 92 || delimiters.include?(value)
+      if mode == :word && whitespace_byte?(value)
+        return escaped
+      end
+      "\\" + escaped
+    when :double
+      double_quoted_escape(value, escaped, start, quote: quote, delimiters: delimiters)
+    else
+      fail!("unsupported escape mode", start)
+    end
+  end
+
+  def double_quoted_escape(value, escaped, start, quote:, delimiters:)
+    return "" if value == 10
+    if value == 13
+      advance if byte == 10
+      return ""
+    end
     return "\n" if value == 110
     return "\t" if value == 116
     return "\r" if value == 114
     return "\f" if value == 102
     return "\a" if value == 97
     return "\e" if value == 101
-    return value.chr if value == 92 || value == 34 || value == 39
-    value.chr
+    return "\b" if value == 98
+    return " " if value == 115
+    return "\v" if value == 118
+    return escaped if value == 92 || value == quote || delimiters.include?(value) || value == 34 || value == 39 || value == 96
+    return hexadecimal_escape(start) if value == 120
+    return octal_escape(value) if value.between?(48, 55)
+    return unicode_escape(start) if value == 117
+    return control_escape(start) if value == 99
+    return control_escape(start, dash: true) if value == 67
+    return meta_escape(start) if value == 77
+    escaped
+  end
+
+  def hexadecimal_escape(start)
+    digits = +""
+    2.times do
+      break unless hex_digit?(byte)
+      digits << byte.chr
+      advance
+    end
+    fail!("invalid hexadecimal escape", start) if digits.empty?
+    raw_byte(digits.to_i(16))
+  end
+
+  def octal_escape(first)
+    digits = first.chr
+    2.times do
+      break unless byte && byte.between?(48, 55)
+      digits << byte.chr
+      advance
+    end
+    raw_byte(digits.to_i(8) & 0xff)
+  end
+
+  def unicode_escape(start)
+    codepoints = []
+    if byte == 123
+      advance
+      loop do
+        advance while whitespace_byte?(byte)
+        digits = +""
+        while hex_digit?(byte)
+          digits << byte.chr
+          advance
+        end
+        fail!("invalid Unicode escape", start) if digits.empty? || digits.length > 6
+        codepoints << unicode_codepoint(digits.to_i(16), start)
+        advance while whitespace_byte?(byte)
+        break if byte == 125
+        fail!("invalid Unicode escape", start) unless byte
+      end
+      advance
+    else
+      4.times do
+        fail!("invalid Unicode escape", start) unless hex_digit?(byte)
+        codepoints << (byte.chr)
+        advance
+      end
+      codepoints = [unicode_codepoint(codepoints.join.to_i(16), start)]
+    end
+    codepoints.join
+  end
+
+  def unicode_codepoint(value, start)
+    fail!("invalid Unicode codepoint", start) if value > 0x10ffff || value.between?(0xd800, 0xdfff)
+    value.chr(Encoding::UTF_8)
+  end
+
+  def control_escape(start, dash: false)
+    if dash
+      fail!("invalid control escape", start) unless byte == 45
+      advance
+    end
+    fail!("unsupported compound control escape", start) if byte == 92
+    character = source_character
+    fail!("invalid control escape", start) unless character.bytesize == 1 && character.getbyte(0) < 128
+    value = character.getbyte(0)
+    raw_byte(value == 63 ? 127 : value & 0x9f)
+  end
+
+  def meta_escape(start)
+    fail!("invalid meta escape", start) unless byte == 45
+    advance
+    fail!("unsupported compound meta escape", start) if byte == 92
+    character = source_character
+    fail!("invalid meta escape", start) unless character.bytesize == 1 && character.getbyte(0) < 128
+    raw_byte(character.getbyte(0) | 0x80)
+  end
+
+  def raw_byte(value)
+    encoding = @source_encoding == Encoding::US_ASCII.name && value >= 0x80 ?
+      Encoding::BINARY : Encoding::UTF_8
+    [value].pack("C").force_encoding(encoding)
+  end
+
+  def hex_digit?(value)
+    value && (value.between?(48, 57) || value.between?(65, 70) || value.between?(97, 102))
+  end
+
+  def whitespace_byte?(value)
+    [9, 10, 11, 12, 13, 32].include?(value)
   end
 
   def regexp_or_operator(start)
-    return operator_or_punctuation(start) if [".", :tCOLON2, :tANDDOT, :keyword_def, :tSYMBEG].include?(@previous)
-    if @context.begin_expression
+    return operator_or_punctuation(start) if [".", :tCOLON2, :tANDDOT, :keyword_def, :tSYMBEG].include?(@previous) ||
+      super_binary_operator?(start, 1)
+    if @context.begin_expression || regexp_command_argument?
       advance
-      value = read_regexp(start)
+      value, metadata = read_regexp(start)
       content = value.is_a?(Array) ? value : [[:tSTRING_CONTENT, value]]
-      @pending = content + [[:tREGEXP_END, nil]]
+      @pending = content + [[:tREGEXP_END, metadata]]
       return [:tREGEXP_BEG, nil]
     end
     operator_or_punctuation(start)
@@ -937,9 +1217,9 @@ class Lexer
     until eof?
       value = byte
       if value == 92
-        result << value.chr
         advance
         fail!("unterminated regexp", start) if eof?
+        result << "\\" unless byte == 47
         result << byte.chr
         advance
       elsif value == 91
@@ -950,22 +1230,38 @@ class Lexer
         in_class = false
         result << value.chr
         advance
-      elsif value == 35 && byte(1) == 123 && !in_class
-        tokens << [:tSTRING_CONTENT, result] unless result.empty?
+      elsif value == 35 && byte(1) == 123
+        tokens << [:tSTRING_CONTENT, result.dup.force_encoding(@source_encoding)] unless result.empty?
         result = +""
         interpolated = true
         advance(2)
+        expression_line = @line
         expression = read_interpolation_source(start)
-        inner = self.class.new(expression, filename: @filename).each.to_a
+        inner = self.class.new(expression, filename: @filename, initial_line: expression_line).each.to_a
         inner.pop if inner.last == [0, nil]
+        inner.map! { |token| token.first == "\n" ? [";", nil] : token }
         tokens << [:tSTRING_DBEG, nil]
         tokens.concat(inner)
         tokens << [:tSTRING_DEND, nil]
+      elsif value == 35 && (length = shorthand_interpolation_length)
+        tokens << [:tSTRING_CONTENT, result.dup.force_encoding(@source_encoding)] unless result.empty?
+        result = +""
+        interpolated = true
+        tokens.concat(shorthand_interpolation_tokens(start, length))
       elsif value == 47 && !in_class
         advance
-        advance while byte && byte.between?(97, 122)
+        metadata = read_regexp_options
+        result.scan(/\(\?<([A-Za-z_][A-Za-z0-9_]*)>/) do
+          @local_variables[Regexp.last_match(1).to_sym] = true
+        end
+        result.force_encoding(@source_encoding)
         tokens << [:tSTRING_CONTENT, result] unless result.empty?
-        return interpolated ? tokens : result
+        if metadata[:encoding] == "US-ASCII"
+          tokens.each do |token|
+            token[1].force_encoding(Encoding::BINARY) if token[0] == :tSTRING_CONTENT
+          end
+        end
+        return [interpolated ? tokens : result, metadata]
       else
         result << value.chr
         advance
@@ -974,6 +1270,27 @@ class Lexer
     fail!("unterminated regexp", start)
   end
 
+  def read_regexp_options
+    modifiers = +""
+    while byte && byte.between?(97, 122)
+      modifiers << byte.chr
+      advance
+    end
+    options = 0
+    options |= 1 if modifiers.include?("i")
+    options |= 2 if modifiers.include?("x")
+    options |= 4 if modifiers.include?("m")
+    encoding = case modifiers.chars.find { |modifier| ["n", "e", "s", "u"].include?(modifier) }
+    when "n" then "US-ASCII"
+    when "e" then "EUC-JP"
+    when "s" then "Windows-31J"
+    when "u" then "UTF-8"
+    end
+    options |= encoding == "US-ASCII" ? 32 : 16 if encoding
+    { options: options, encoding: encoding, source_encoding: @source_encoding,
+      once: modifiers.include?("o") }.freeze
+  end
+
   def percent_token(start)
     return operator_or_punctuation(start) if [".", :tCOLON2, :tANDDOT].include?(@previous)
     literal_kind = [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(byte(1))
@@ -996,42 +1313,92 @@ class Lexer
     advance
     closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter)
     interpolate = [81, 87, 73, 114, 120, 88].include?(kind)
-    content = interpolate ?
-      read_interpolated_delimited(closing, start) :
-      read_delimited(closing, start, interpolate: interpolate)
-    advance while kind == 114 && byte && byte.between?(97, 122)
-    token = { 113 => :tSTRING_BEG, 81 => :tSTRING_BEG, 119 => :tWORDS_BEG, 87 => :tQWORDS_BEG,
-              105 => :tSYMBOLS_BEG, 73 => :tQSYMBOLS_BEG, 114 => :tREGEXP_BEG,
+    word_literal = [119, 87, 105, 73].include?(kind)
+    content = if word_literal
+      read_word_delimited(closing, delimiter, start, interpolate: [87, 73].include?(kind), symbols: [105, 73].include?(kind))
+    elsif interpolate
+      mode = kind == 114 ? :regexp : :double
+      read_interpolated_delimited(closing, start, mode: mode, delimiters: [closing, delimiter])
+    else
+      read_delimited(closing, start, mode: :percent_q, delimiters: [closing, delimiter])
+    end
+    regexp_metadata = read_regexp_options if kind == 114
+    token = { 113 => :tSTRING_BEG, 81 => :tSTRING_BEG, 87 => :tWORDS_BEG, 119 => :tWORDS_BEG,
+              73 => :tSYMBOLS_BEG, 105 => :tSYMBOLS_BEG, 114 => :tREGEXP_BEG,
               115 => :tSYMBEG, 120 => :tXSTRING_BEG, 88 => :tXSTRING_BEG }.fetch(kind)
     terminator = kind == 114 ? :tREGEXP_END : :tSTRING_END
-    @pending = if [119, 87, 105, 73].include?(kind)
-      content = content.map { |token, value| value.to_s }.join if content.is_a?(Array)
-      word_tokens(content, symbols: [105, 73].include?(kind)) + [[terminator, nil]]
-    elsif content.is_a?(Array)
+    @pending = if word_literal
       content + [[terminator, nil]]
+    elsif content.is_a?(Array)
+      content + [[terminator, regexp_metadata]]
     else
-      [[:tSTRING_CONTENT, content], [terminator, nil]]
+      [[:tSTRING_CONTENT, content], [terminator, regexp_metadata]]
     end
     [token, nil]
   end
 
-  def word_tokens(content, symbols:)
-    words = content.split(/[\t\n\f\r ]+/).reject(&:empty?)
+  def read_word_delimited(closing, opening, start, interpolate:, symbols:)
     tokens = [[" ", nil]]
-    words.each do |word|
-      tokens << [:tSTRING_CONTENT, symbols ? word.to_sym : word]
-      tokens << [" ", nil]
+    content = +""
+    word_started = false
+    depth = 0
+    paired_opening = { 41 => 40, 93 => 91, 125 => 123, 62 => 60 }.fetch(closing, -1)
+    flush_content = lambda do
+      unless content.empty?
+        tokens << [:tSTRING_CONTENT, content]
+        content = +""
+      end
     end
-    tokens
+    flush_word = lambda do
+      flush_content.call
+      if word_started
+        tokens << [" ", nil]
+        word_started = false
+      end
+    end
+
+    until eof?
+      value = byte
+      if value == 92
+        content << escape_sequence(start, mode: interpolate ? :double : :word,
+          delimiters: [closing, opening])
+        word_started = true
+      elsif value == 35 && interpolate && byte(1) == 123
+        flush_content.call
+        word_started = true
+        advance(2)
+        expression_line = @line
+        expression = read_interpolation_source(start)
+        inner = self.class.new(expression, filename: @filename, initial_line: expression_line).each.to_a
+        inner.pop if inner.last == [0, nil]
+        inner.map! { |token| token.first == "\n" ? [";", nil] : token }
+        tokens << [:tSTRING_DBEG, nil]
+        tokens.concat(inner)
+        tokens << [:tSTRING_DEND, nil]
+      elsif value == closing && depth.zero?
+        flush_word.call
+        advance
+        return tokens
+      elsif whitespace_byte?(value)
+        flush_word.call
+        advance
+      else
+        depth += 1 if value == paired_opening
+        depth -= 1 if value == closing && depth.positive?
+        append_source_character(content)
+        word_started = true
+      end
+    end
+    fail!("unterminated percent literal", start)
   end
 
-  def read_delimited(closing, start, interpolate: false)
+  def read_delimited(closing, start, mode:, delimiters: [], interpolate: false)
     result = +""
     depth = 0
     until eof?
       value = byte
       if value == 92
-        result << escape_sequence(start)
+        result << escape_sequence(start, mode: mode, delimiters: delimiters)
       elsif value == 35 && interpolate && byte(1) == 123
         fail!("string interpolation cannot be represented by the AST", start)
       elsif value == closing && depth.zero?
@@ -1040,14 +1407,13 @@ class Lexer
       else
         depth += 1 if value == ({ 41 => 40, 93 => 91, 125 => 123, 62 => 60 }.fetch(closing, -1))
         depth -= 1 if value == closing && depth.positive?
-        result << value.chr
-        advance
+        append_source_character(result)
       end
     end
     fail!("unterminated percent literal", start)
   end
 
-  def read_interpolated_delimited(closing, start)
+  def read_interpolated_delimited(closing, start, mode:, delimiters: [])
     result = +""
     tokens = []
     depth = 0
@@ -1055,7 +1421,7 @@ class Lexer
     until eof?
       value = byte
       if value == 92
-        result << escape_sequence(start)
+        result << escape_sequence(start, mode: mode, delimiters: delimiters)
       elsif value == 35 && byte(1) == 123
         tokens << [:tSTRING_CONTENT, result] unless result.empty?
         result = +""
@@ -1063,56 +1429,114 @@ class Lexer
         expression = read_interpolation_source(start)
         inner = self.class.new(expression, filename: @filename).each.to_a
         inner.pop if inner.last == [0, nil]
+        inner.map! { |token| token.first == "\n" ? [";", nil] : token }
         tokens << [:tSTRING_DBEG, nil]
         tokens.concat(inner)
         tokens << [:tSTRING_DEND, nil]
       elsif value == closing && depth.zero?
         advance
-        tokens << [:tSTRING_CONTENT, result] unless result.empty?
+        tokens << [:tSTRING_CONTENT, result] if tokens.empty? || !result.empty?
         return tokens
       else
         depth += 1 if value == opening
         depth -= 1 if value == closing && depth.positive?
-        result << value.chr
-        advance
+        append_source_character(result)
       end
     end
     fail!("unterminated percent literal", start)
   end
 
   def interpolated_content_tokens(source, start)
-    return [[:tSTRING_CONTENT, source]] unless source.include?("#" + "{")
+    lexer = self.class.new(source, filename: @filename)
+    lexer.instance_variable_set(:@source_encoding, @source_encoding)
+    lexer.send(:read_interpolated_body, start)
+  end
+
+  def read_interpolated_body(start)
     tokens = []
-    literal = +""
-    index = 0
-    while index < source.bytesize
-      if source.getbyte(index) == 35 && source.getbyte(index + 1) == 123
+    literal = source_string
+    until eof?
+      value = byte
+      if value == 92
+        literal << escape_sequence(start, mode: :double)
+      elsif value == 35 && byte(1) == 123
         tokens << [:tSTRING_CONTENT, literal] unless literal.empty?
-        depth = 1
-        expr_start = index + 2
-        index = expr_start
-        while index < source.bytesize && depth.positive?
-          depth += 1 if source.getbyte(index) == 123
-          depth -= 1 if source.getbyte(index) == 125
-          index += 1
-        end
-        fail!("unterminated string interpolation", start) unless depth.zero?
-        expression = source.byteslice(expr_start, index - expr_start - 1)
+        advance(2)
+        expression = read_interpolation_source(start)
         inner = self.class.new(expression, filename: @filename).each.to_a
         inner.pop if inner.last == [0, nil]
+        inner.map! { |token| token.first == "\n" ? [";", nil] : token }
         tokens << [:tSTRING_DBEG, nil]
         tokens.concat(inner)
         tokens << [:tSTRING_DEND, nil]
-        literal = +""
+        literal = source_string
+      elsif value == 35 && (length = shorthand_interpolation_length)
+        tokens << [:tSTRING_CONTENT, literal] unless literal.empty?
+        literal = source_string
+        tokens.concat(shorthand_interpolation_tokens(start, length))
       else
-        literal << source.getbyte(index).chr
-        index += 1
+        append_source_character(literal)
       end
     end
     tokens << [:tSTRING_CONTENT, literal] unless literal.empty?
     tokens
   end
 
+  def shorthand_interpolation_tokens(start, length)
+    expression_line = @line
+    expression = @source.byteslice(@index + 1, length).force_encoding(Encoding::UTF_8)
+    advance(length + 1)
+    inner = self.class.new(expression, filename: @filename, initial_line: expression_line).each.to_a
+    inner.pop if inner.last == [0, nil]
+    [[:tSTRING_DBEG, nil], *inner, [:tSTRING_DEND, nil]]
+  end
+
+  def shorthand_interpolation_length
+    return unless byte == 35
+
+    case byte(1)
+    when 64
+      offset = 2
+      offset += 1 if byte(offset) == 64
+      return unless identifier_start_byte?(byte(offset))
+      offset += 1 while identifier_byte?(byte(offset))
+      offset - 1
+    when 36
+      offset = 2
+      if byte(offset) == 45
+        return unless identifier_start_byte?(byte(offset + 1))
+        offset += 2
+        offset += 1 while identifier_byte?(byte(offset))
+      elsif digit_byte?(byte(offset), 10)
+        offset += 1
+        offset += 1 while digit_byte?(byte(offset), 10)
+      elsif identifier_start_byte?(byte(offset))
+        offset += 1
+        offset += 1 while identifier_byte?(byte(offset))
+      elsif [33, 36, 38, 39, 42, 43, 44, 45, 46, 47, 58, 59, 60, 61, 62, 63, 64, 92, 96, 126].include?(byte(offset))
+        offset += 1
+      else
+        return
+      end
+      offset - 1
+    end
+  end
+
+  def identifier_start_byte?(value)
+    value && (value == 95 || value.between?(65, 90) || value.between?(97, 122) || value >= 128)
+  end
+
+  def source_local_variable_names
+    @source.scan(/(?<![.@:$])([a-z_][a-zA-Z0-9_]*)[ \t\r\n]*[+\-*\/%]?=(?!=)/).flatten.map(&:to_sym).uniq
+  end
+
+  def end_data_marker?(start)
+    line_start = @source.rindex("\n".b, start - 1)
+    line_start = line_start ? line_start + 1 : 0
+    prefix = @source.byteslice(line_start, start - line_start)
+    prefix.bytes.all? { |value| [9, 13, 32].include?(value) }
+  end
+
   def heredoc_or_operator(start)
     return operator_or_punctuation(start) if [".", :keyword_class].include?(@previous)
     if byte(1) == 60 && heredoc_prefix?
@@ -1137,6 +1561,7 @@ class Lexer
   end
 
   def heredoc_token(start)
+    suffix_line = @line
     advance(2)
     indent = byte == 45 || byte == 126
     squiggly = byte == 126
@@ -1165,11 +1590,14 @@ class Lexer
     @pending = content_tokens + [[:tSTRING_END, nil]]
     prefix, headers, suffix = extract_heredoc_headers(suffix)
     unless prefix.empty?
-      prefix_tokens = heredoc_suffix_tokens(prefix)
+      prefix_tokens = heredoc_suffix_tokens(prefix, suffix_line)
       @pending.concat(prefix_tokens)
     end
     @heredoc_queue.concat(headers)
-    @deferred_suffix = suffix unless suffix.empty?
+    unless suffix.empty?
+      @deferred_suffix = suffix
+      @deferred_suffix_line = suffix_line
+    end
     @heredoc_newline_pending = true
     [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil]
   end
@@ -1190,7 +1618,7 @@ class Lexer
       advance if byte == 10
       fail!("unterminated heredoc", start) if eof?
     end
-    body = body.force_encoding(Encoding::UTF_8)
+    body = body.force_encoding(@source_encoding)
     squiggly ? dedent_heredoc(body) : body
   end
 
@@ -1207,8 +1635,9 @@ class Lexer
     [suffix[0...match.begin(0)], headers, trailing]
   end
 
-  def heredoc_suffix_tokens(source)
+  def heredoc_suffix_tokens(source, line = @line)
     suffix_lexer = self.class.new(source, filename: @filename)
+    suffix_lexer.instance_variable_set(:@line, line)
     suffix_lexer.context.begin_expression = false
     suffix_lexer.context.lex_state = :expr_end
     suffix_lexer.instance_variable_set(:@previous, :tSTRING_END)
@@ -1249,12 +1678,12 @@ class Lexer
     if @previous == :keyword_def && ["-@", "+@"].include?(@source.byteslice(@index, 2))
       value = @source.byteslice(@index, 2)
       advance(2)
-      return [value == "-@" ? :tUMINUS : :tUPLUS, nil]
+      return [value == "-@" ? :tUMINUS : :tUPLUS, value]
     end
     if (@previous == :tSYMBEG || @context.alias_context) && ["-@", "+@"].include?(@source.byteslice(@index, 2))
       value = @source.byteslice(@index, 2)
       advance(2)
-      return [value == "-@" ? :tUMINUS : :tUPLUS, nil]
+      return [value == "-@" ? :tUMINUS : :tUPLUS, value]
     end
     text = OPERATORS.sort_by { |operator| -operator.bytesize }.find do |operator|
       next false if ["[]", "[]="].include?(operator) && start.positive? &&
@@ -1271,7 +1700,7 @@ class Lexer
       @previous_previous == :keyword_alias
     if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text))
       advance(text.bytesize)
-      token = if text == "**" && @context.begin_expression
+      token = if text == "**" && @context.begin_expression && !super_binary_operator?(start, text.bytesize)
         :tDSTAR
       elsif text == ".." && @context.begin_expression
         :tBDOT2
@@ -1344,11 +1773,18 @@ class Lexer
         @context.method_body_start = true
         @context.method_header = false
       end
-    elsif value == "-" && @context.begin_expression
-      return [:tUMINUS, nil]
-    elsif value == "+" && @context.begin_expression
-      return [:tUPLUS, nil]
-    elsif value == "*" && @context.begin_expression
+    elsif value == "-" && @previous == :tSYMBEG
+      return [:tUMINUS, :-]
+    elsif value == "+" && @previous == :tSYMBEG
+      return [:tUPLUS, :+]
+    elsif value == "-" && (@context.begin_expression || unicode_identifier_command_argument?(start) || unary_argument_without_space?(start)) && !super_binary_operator?(start, 1)
+      token = numeric_literal_follows? ? :tUMINUS_NUM : :tUMINUS
+      return [token, @previous == :keyword_def || @context.alias_context ? value : nil]
+    elsif value == "+" && (@context.begin_expression || unicode_identifier_command_argument?(start) || unary_argument_without_space?(start)) && !super_binary_operator?(start, 1)
+      return [:tUPLUS, @previous == :keyword_def || @context.alias_context ? value : nil]
+    elsif value == "*" && @context.begin_expression && !super_binary_operator?(start, 1)
+      return [:tSTAR, nil]
+    elsif value == "*" && splat_argument_without_space?(start)
       return [:tSTAR, nil]
     elsif value == "&" && @previous == :tSYMBEG
       return [value, nil]
@@ -1366,6 +1802,47 @@ class Lexer
     fail!("invalid byte", start)
   end
 
+  def super_binary_operator?(start, length)
+    return false unless @previous == :keyword_super
+
+    before = start.positive? ? @source.getbyte(start - 1) : nil
+    after = @source.getbyte(start + length)
+    !whitespace_byte?(before) || whitespace_byte?(after)
+  end
+
+  def regexp_command_argument?
+    @previous == :tIDENTIFIER && @previous_previous == :modifier_if &&
+      !@local_variables.key?(@previous_value) &&
+      !whitespace_byte?(byte(1))
+  end
+
+  def unicode_identifier_command_argument?(start)
+    return false unless @previous == :tIDENTIFIER
+    return false unless @previous_value&.to_s&.match?(Regexp.new("\\p{Cf}"))
+
+    before = start.positive? ? @source.getbyte(start - 1) : nil
+    after = @source.getbyte(start + 1)
+    whitespace_byte?(before) && !whitespace_byte?(after)
+  end
+
+  def numeric_literal_follows?
+    value = byte
+    value && value.between?(48, 57)
+  end
+
+  def splat_argument_without_space?(start)
+    return false unless whitespace_byte?(@source.getbyte(start - 1))
+    return false if whitespace_byte?(@source.getbyte(start + 1))
+    return false if @previous == :tIDENTIFIER &&
+      (@local_variables.key?(@previous_value) || @source_local_variables.include?(@previous_value))
+    [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous)
+  end
+
+  def unary_argument_without_space?(start)
+    return false if whitespace_byte?(@source.getbyte(start + 1))
+    splat_argument_without_space?(start)
+  end
+
   def begin_expression?
     @context.begin_expression
   end