prism 0.13.0

data/lib/prism/serialize.rb

@@ -0,0 +1,662 @@
+# frozen_string_literal: true
+=begin
+This file is generated by the templates/template.rb script and should not be
+modified manually. See templates/lib/prism/serialize.rb.erb
+if you are looking to modify the template
+=end
+
+require "stringio"
+
+# Polyfill for String#unpack1 with the offset parameter.
+if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
+  String.prepend(
+    Module.new {
+      def unpack1(format, offset: 0)
+        offset == 0 ? super(format) : self[offset..].unpack1(format)
+      end
+    }
+  )
+end
+
+module Prism
+  module Serialize
+    MAJOR_VERSION = 0
+    MINOR_VERSION = 13
+    PATCH_VERSION = 0
+
+    def self.load(input, serialized)
+      input = input.dup
+      source = Source.new(input)
+      loader = Loader.new(source, serialized)
+      result = loader.load_result
+
+      input.force_encoding(loader.encoding)
+      result
+    end
+
+    def self.load_tokens(source, serialized)
+      Loader.new(source, serialized).load_tokens_result
+    end
+
+    class Loader
+      attr_reader :encoding, :input, :serialized, :io
+      attr_reader :constant_pool_offset, :constant_pool, :source
+
+      def initialize(source, serialized)
+        @encoding = Encoding::UTF_8
+
+        @input = source.source.dup
+        @serialized = serialized
+        @io = StringIO.new(serialized)
+        @io.set_encoding(Encoding::BINARY)
+
+        @constant_pool_offset = nil
+        @constant_pool = nil
+
+        @source = source
+      end
+
+      def load_encoding
+        Encoding.find(io.read(load_varint))
+      end
+
+      def load_metadata
+        comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) }
+        errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) }
+        warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) }
+        [comments, errors, warnings]
+      end
+
+      def load_tokens
+        tokens = []
+        while type = TOKEN_TYPES.fetch(load_varint)
+          start = load_varint
+          length = load_varint
+          lex_state = load_varint
+          location = Location.new(@source, start, length)
+          tokens << [Prism::Token.new(type, location.slice, location), lex_state]
+        end
+
+        tokens
+      end
+
+      def load_tokens_result
+        tokens = load_tokens
+        encoding = load_encoding
+        comments, errors, warnings = load_metadata
+
+        if encoding != @encoding
+          tokens.each { |token,| token.value.force_encoding(encoding) }
+        end
+
+        raise "Expected to consume all bytes while deserializing" unless @io.eof?
+        Prism::ParseResult.new(tokens, comments, errors, warnings, @source)
+      end
+
+      def load_nodes
+        raise "Invalid serialization" if io.read(5) != "PRISM"
+        raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
+        only_semantic_fields = io.read(1).unpack1("C")
+        unless only_semantic_fields == 0
+          raise "Invalid serialization (location fields must be included but are not)"
+        end
+
+        @encoding = load_encoding
+        @input = input.force_encoding(@encoding).freeze
+
+        comments, errors, warnings = load_metadata
+
+        @constant_pool_offset = io.read(4).unpack1("L")
+        @constant_pool = Array.new(load_varint, nil)
+
+        [load_node, comments, errors, warnings]
+      end
+
+      def load_result
+        node, comments, errors, warnings = load_nodes
+        Prism::ParseResult.new(node, comments, errors, warnings, @source)
+      end
+
+      private
+
+      # variable-length integer using https://en.wikipedia.org/wiki/LEB128
+      # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
+      def load_varint
+        n = io.getbyte
+        if n < 128
+          n
+        else
+          n -= 128
+          shift = 0
+          while (b = io.getbyte) >= 128
+            n += (b - 128) << (shift += 7)
+          end
+          n + (b << (shift + 7))
+        end
+      end
+
+      def load_serialized_length
+        io.read(4).unpack1("L")
+      end
+
+      def load_optional_node
+        if io.getbyte != 0
+          io.pos -= 1
+          load_node
+        end
+      end
+
+      def load_embedded_string
+        io.read(load_varint).force_encoding(encoding)
+      end
+
+      def load_string
+        case io.getbyte
+        when 1
+          input.byteslice(load_varint, load_varint).force_encoding(encoding)
+        when 2
+          load_embedded_string
+        else
+          raise
+        end
+      end
+
+      def load_location
+        Location.new(source, load_varint, load_varint)
+      end
+
+      def load_optional_location
+        load_location if io.getbyte != 0
+      end
+
+      def load_constant(index)
+        constant = constant_pool[index]
+
+        unless constant
+          offset = constant_pool_offset + index * 8
+          start = serialized.unpack1("L", offset: offset)
+          length = serialized.unpack1("L", offset: offset + 4)
+
+          constant =
+            if start.nobits?(1 << 31)
+              input.byteslice(start, length).to_sym
+            else
+              serialized.byteslice(start & ((1 << 31) - 1), length).to_sym
+            end
+
+          constant_pool[index] = constant
+        end
+
+        constant
+      end
+
+      def load_required_constant
+        load_constant(load_varint - 1)
+      end
+
+      def load_optional_constant
+        index = load_varint
+        load_constant(index - 1) if index != 0
+      end
+
+      def load_node
+        type = io.getbyte
+        location = load_location
+
+        case type
+        when 1 then
+          AliasGlobalVariableNode.new(load_node, load_node, load_location, location)
+        when 2 then
+          AliasMethodNode.new(load_node, load_node, load_location, location)
+        when 3 then
+          AlternationPatternNode.new(load_node, load_node, load_location, location)
+        when 4 then
+          AndNode.new(load_node, load_node, load_location, location)
+        when 5 then
+          ArgumentsNode.new(Array.new(load_varint) { load_node }, location)
+        when 6 then
+          ArrayNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
+        when 7 then
+          ArrayPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
+        when 8 then
+          AssocNode.new(load_node, load_optional_node, load_optional_location, location)
+        when 9 then
+          AssocSplatNode.new(load_optional_node, load_location, location)
+        when 10 then
+          BackReferenceReadNode.new(location)
+        when 11 then
+          BeginNode.new(load_optional_location, load_optional_node, load_optional_node, load_optional_node, load_optional_node, load_optional_location, location)
+        when 12 then
+          BlockArgumentNode.new(load_optional_node, load_location, location)
+        when 13 then
+          BlockLocalVariableNode.new(load_required_constant, location)
+        when 14 then
+          BlockNode.new(Array.new(load_varint) { load_required_constant }, load_optional_node, load_optional_node, load_location, load_location, location)
+        when 15 then
+          BlockParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
+        when 16 then
+          BlockParametersNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
+        when 17 then
+          BreakNode.new(load_optional_node, load_location, location)
+        when 18 then
+          CallAndWriteNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_varint, load_string, load_string, load_location, load_node, location)
+        when 19 then
+          CallNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, load_varint, load_string, location)
+        when 20 then
+          CallOperatorWriteNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_varint, load_string, load_string, load_required_constant, load_location, load_node, location)
+        when 21 then
+          CallOrWriteNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_varint, load_string, load_string, load_location, load_node, location)
+        when 22 then
+          CapturePatternNode.new(load_node, load_node, load_location, location)
+        when 23 then
+          CaseNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_location, load_location, location)
+        when 24 then
+          ClassNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_node, load_optional_location, load_optional_node, load_optional_node, load_location, load_required_constant, location)
+        when 25 then
+          ClassVariableAndWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 26 then
+          ClassVariableOperatorWriteNode.new(load_required_constant, load_location, load_location, load_node, load_required_constant, location)
+        when 27 then
+          ClassVariableOrWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 28 then
+          ClassVariableReadNode.new(load_required_constant, location)
+        when 29 then
+          ClassVariableTargetNode.new(load_required_constant, location)
+        when 30 then
+          ClassVariableWriteNode.new(load_required_constant, load_location, load_node, load_optional_location, location)
+        when 31 then
+          ConstantAndWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 32 then
+          ConstantOperatorWriteNode.new(load_required_constant, load_location, load_location, load_node, load_required_constant, location)
+        when 33 then
+          ConstantOrWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 34 then
+          ConstantPathAndWriteNode.new(load_node, load_location, load_node, location)
+        when 35 then
+          ConstantPathNode.new(load_optional_node, load_node, load_location, location)
+        when 36 then
+          ConstantPathOperatorWriteNode.new(load_node, load_location, load_node, load_required_constant, location)
+        when 37 then
+          ConstantPathOrWriteNode.new(load_node, load_location, load_node, location)
+        when 38 then
+          ConstantPathTargetNode.new(load_optional_node, load_node, load_location, location)
+        when 39 then
+          ConstantPathWriteNode.new(load_node, load_location, load_node, location)
+        when 40 then
+          ConstantReadNode.new(load_required_constant, location)
+        when 41 then
+          ConstantTargetNode.new(load_required_constant, location)
+        when 42 then
+          ConstantWriteNode.new(load_required_constant, load_location, load_node, load_location, location)
+        when 43 then
+          load_serialized_length
+          DefNode.new(load_required_constant, load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varint) { load_required_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
+        when 44 then
+          DefinedNode.new(load_optional_location, load_node, load_optional_location, load_location, location)
+        when 45 then
+          ElseNode.new(load_location, load_optional_node, load_optional_location, location)
+        when 46 then
+          EmbeddedStatementsNode.new(load_location, load_optional_node, load_location, location)
+        when 47 then
+          EmbeddedVariableNode.new(load_location, load_node, location)
+        when 48 then
+          EnsureNode.new(load_location, load_optional_node, load_location, location)
+        when 49 then
+          FalseNode.new(location)
+        when 50 then
+          FindPatternNode.new(load_optional_node, load_node, Array.new(load_varint) { load_node }, load_node, load_optional_location, load_optional_location, location)
+        when 51 then
+          FlipFlopNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
+        when 52 then
+          FloatNode.new(location)
+        when 53 then
+          ForNode.new(load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
+        when 54 then
+          ForwardingArgumentsNode.new(location)
+        when 55 then
+          ForwardingParameterNode.new(location)
+        when 56 then
+          ForwardingSuperNode.new(load_optional_node, location)
+        when 57 then
+          GlobalVariableAndWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 58 then
+          GlobalVariableOperatorWriteNode.new(load_required_constant, load_location, load_location, load_node, load_required_constant, location)
+        when 59 then
+          GlobalVariableOrWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 60 then
+          GlobalVariableReadNode.new(load_required_constant, location)
+        when 61 then
+          GlobalVariableTargetNode.new(load_required_constant, location)
+        when 62 then
+          GlobalVariableWriteNode.new(load_required_constant, load_location, load_node, load_location, location)
+        when 63 then
+          HashNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
+        when 64 then
+          HashPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_location, load_optional_location, location)
+        when 65 then
+          IfNode.new(load_optional_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
+        when 66 then
+          ImaginaryNode.new(load_node, location)
+        when 67 then
+          ImplicitNode.new(load_node, location)
+        when 68 then
+          InNode.new(load_node, load_optional_node, load_location, load_optional_location, location)
+        when 69 then
+          InstanceVariableAndWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 70 then
+          InstanceVariableOperatorWriteNode.new(load_required_constant, load_location, load_location, load_node, load_required_constant, location)
+        when 71 then
+          InstanceVariableOrWriteNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 72 then
+          InstanceVariableReadNode.new(load_required_constant, location)
+        when 73 then
+          InstanceVariableTargetNode.new(load_required_constant, location)
+        when 74 then
+          InstanceVariableWriteNode.new(load_required_constant, load_location, load_node, load_location, location)
+        when 75 then
+          IntegerNode.new(load_varint, location)
+        when 76 then
+          InterpolatedMatchLastLineNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
+        when 77 then
+          InterpolatedRegularExpressionNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
+        when 78 then
+          InterpolatedStringNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
+        when 79 then
+          InterpolatedSymbolNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
+        when 80 then
+          InterpolatedXStringNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
+        when 81 then
+          KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
+        when 82 then
+          KeywordParameterNode.new(load_required_constant, load_location, load_optional_node, location)
+        when 83 then
+          KeywordRestParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
+        when 84 then
+          LambdaNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_location, load_location, load_optional_node, load_optional_node, location)
+        when 85 then
+          LocalVariableAndWriteNode.new(load_location, load_location, load_node, load_required_constant, load_varint, location)
+        when 86 then
+          LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_required_constant, load_required_constant, load_varint, location)
+        when 87 then
+          LocalVariableOrWriteNode.new(load_location, load_location, load_node, load_required_constant, load_varint, location)
+        when 88 then
+          LocalVariableReadNode.new(load_required_constant, load_varint, location)
+        when 89 then
+          LocalVariableTargetNode.new(load_required_constant, load_varint, location)
+        when 90 then
+          LocalVariableWriteNode.new(load_required_constant, load_varint, load_location, load_node, load_location, location)
+        when 91 then
+          MatchLastLineNode.new(load_location, load_location, load_location, load_string, load_varint, location)
+        when 92 then
+          MatchPredicateNode.new(load_node, load_node, load_location, location)
+        when 93 then
+          MatchRequiredNode.new(load_node, load_node, load_location, location)
+        when 94 then
+          MatchWriteNode.new(load_node, Array.new(load_varint) { load_required_constant }, location)
+        when 95 then
+          MissingNode.new(location)
+        when 96 then
+          ModuleNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_node, load_optional_node, load_location, load_required_constant, location)
+        when 97 then
+          MultiTargetNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
+        when 98 then
+          MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, load_location, load_node, location)
+        when 99 then
+          NextNode.new(load_optional_node, load_location, location)
+        when 100 then
+          NilNode.new(location)
+        when 101 then
+          NoKeywordsParameterNode.new(load_location, load_location, location)
+        when 102 then
+          NumberedReferenceReadNode.new(load_varint, location)
+        when 103 then
+          OptionalParameterNode.new(load_required_constant, load_location, load_location, load_node, location)
+        when 104 then
+          OrNode.new(load_node, load_node, load_location, location)
+        when 105 then
+          ParametersNode.new(Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, load_optional_node, location)
+        when 106 then
+          ParenthesesNode.new(load_optional_node, load_location, load_location, location)
+        when 107 then
+          PinnedExpressionNode.new(load_node, load_location, load_location, load_location, location)
+        when 108 then
+          PinnedVariableNode.new(load_node, load_location, location)
+        when 109 then
+          PostExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
+        when 110 then
+          PreExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
+        when 111 then
+          ProgramNode.new(Array.new(load_varint) { load_required_constant }, load_node, location)
+        when 112 then
+          RangeNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
+        when 113 then
+          RationalNode.new(load_node, location)
+        when 114 then
+          RedoNode.new(location)
+        when 115 then
+          RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
+        when 116 then
+          RequiredDestructuredParameterNode.new(Array.new(load_varint) { load_node }, load_location, load_location, location)
+        when 117 then
+          RequiredParameterNode.new(load_required_constant, location)
+        when 118 then
+          RescueModifierNode.new(load_node, load_location, load_node, location)
+        when 119 then
+          RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
+        when 120 then
+          RestParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
+        when 121 then
+          RetryNode.new(location)
+        when 122 then
+          ReturnNode.new(load_location, load_optional_node, location)
+        when 123 then
+          SelfNode.new(location)
+        when 124 then
+          SingletonClassNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
+        when 125 then
+          SourceEncodingNode.new(location)
+        when 126 then
+          SourceFileNode.new(load_string, location)
+        when 127 then
+          SourceLineNode.new(location)
+        when 128 then
+          SplatNode.new(load_location, load_optional_node, location)
+        when 129 then
+          StatementsNode.new(Array.new(load_varint) { load_node }, location)
+        when 130 then
+          StringConcatNode.new(load_node, load_node, location)
+        when 131 then
+          StringNode.new(load_varint, load_optional_location, load_location, load_optional_location, load_string, location)
+        when 132 then
+          SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
+        when 133 then
+          SymbolNode.new(load_optional_location, load_optional_location, load_optional_location, load_string, location)
+        when 134 then
+          TrueNode.new(location)
+        when 135 then
+          UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
+        when 136 then
+          UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
+        when 137 then
+          UntilNode.new(load_location, load_optional_location, load_node, load_optional_node, load_varint, location)
+        when 138 then
+          WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
+        when 139 then
+          WhileNode.new(load_location, load_optional_location, load_node, load_optional_node, load_varint, location)
+        when 140 then
+          XStringNode.new(load_location, load_location, load_location, load_string, location)
+        when 141 then
+          YieldNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, location)
+        end
+      end
+    end
+
+    TOKEN_TYPES = [
+      nil,
+      :EOF,
+      :MISSING,
+      :NOT_PROVIDED,
+      :AMPERSAND,
+      :AMPERSAND_AMPERSAND,
+      :AMPERSAND_AMPERSAND_EQUAL,
+      :AMPERSAND_DOT,
+      :AMPERSAND_EQUAL,
+      :BACKTICK,
+      :BACK_REFERENCE,
+      :BANG,
+      :BANG_EQUAL,
+      :BANG_TILDE,
+      :BRACE_LEFT,
+      :BRACE_RIGHT,
+      :BRACKET_LEFT,
+      :BRACKET_LEFT_ARRAY,
+      :BRACKET_LEFT_RIGHT,
+      :BRACKET_LEFT_RIGHT_EQUAL,
+      :BRACKET_RIGHT,
+      :CARET,
+      :CARET_EQUAL,
+      :CHARACTER_LITERAL,
+      :CLASS_VARIABLE,
+      :COLON,
+      :COLON_COLON,
+      :COMMA,
+      :COMMENT,
+      :CONSTANT,
+      :DOT,
+      :DOT_DOT,
+      :DOT_DOT_DOT,
+      :EMBDOC_BEGIN,
+      :EMBDOC_END,
+      :EMBDOC_LINE,
+      :EMBEXPR_BEGIN,
+      :EMBEXPR_END,
+      :EMBVAR,
+      :EQUAL,
+      :EQUAL_EQUAL,
+      :EQUAL_EQUAL_EQUAL,
+      :EQUAL_GREATER,
+      :EQUAL_TILDE,
+      :FLOAT,
+      :FLOAT_IMAGINARY,
+      :FLOAT_RATIONAL,
+      :FLOAT_RATIONAL_IMAGINARY,
+      :GLOBAL_VARIABLE,
+      :GREATER,
+      :GREATER_EQUAL,
+      :GREATER_GREATER,
+      :GREATER_GREATER_EQUAL,
+      :HEREDOC_END,
+      :HEREDOC_START,
+      :IDENTIFIER,
+      :IGNORED_NEWLINE,
+      :INSTANCE_VARIABLE,
+      :INTEGER,
+      :INTEGER_IMAGINARY,
+      :INTEGER_RATIONAL,
+      :INTEGER_RATIONAL_IMAGINARY,
+      :KEYWORD_ALIAS,
+      :KEYWORD_AND,
+      :KEYWORD_BEGIN,
+      :KEYWORD_BEGIN_UPCASE,
+      :KEYWORD_BREAK,
+      :KEYWORD_CASE,
+      :KEYWORD_CLASS,
+      :KEYWORD_DEF,
+      :KEYWORD_DEFINED,
+      :KEYWORD_DO,
+      :KEYWORD_DO_LOOP,
+      :KEYWORD_ELSE,
+      :KEYWORD_ELSIF,
+      :KEYWORD_END,
+      :KEYWORD_END_UPCASE,
+      :KEYWORD_ENSURE,
+      :KEYWORD_FALSE,
+      :KEYWORD_FOR,
+      :KEYWORD_IF,
+      :KEYWORD_IF_MODIFIER,
+      :KEYWORD_IN,
+      :KEYWORD_MODULE,
+      :KEYWORD_NEXT,
+      :KEYWORD_NIL,
+      :KEYWORD_NOT,
+      :KEYWORD_OR,
+      :KEYWORD_REDO,
+      :KEYWORD_RESCUE,
+      :KEYWORD_RESCUE_MODIFIER,
+      :KEYWORD_RETRY,
+      :KEYWORD_RETURN,
+      :KEYWORD_SELF,
+      :KEYWORD_SUPER,
+      :KEYWORD_THEN,
+      :KEYWORD_TRUE,
+      :KEYWORD_UNDEF,
+      :KEYWORD_UNLESS,
+      :KEYWORD_UNLESS_MODIFIER,
+      :KEYWORD_UNTIL,
+      :KEYWORD_UNTIL_MODIFIER,
+      :KEYWORD_WHEN,
+      :KEYWORD_WHILE,
+      :KEYWORD_WHILE_MODIFIER,
+      :KEYWORD_YIELD,
+      :KEYWORD___ENCODING__,
+      :KEYWORD___FILE__,
+      :KEYWORD___LINE__,
+      :LABEL,
+      :LABEL_END,
+      :LAMBDA_BEGIN,
+      :LESS,
+      :LESS_EQUAL,
+      :LESS_EQUAL_GREATER,
+      :LESS_LESS,
+      :LESS_LESS_EQUAL,
+      :METHOD_NAME,
+      :MINUS,
+      :MINUS_EQUAL,
+      :MINUS_GREATER,
+      :NEWLINE,
+      :NUMBERED_REFERENCE,
+      :PARENTHESIS_LEFT,
+      :PARENTHESIS_LEFT_PARENTHESES,
+      :PARENTHESIS_RIGHT,
+      :PERCENT,
+      :PERCENT_EQUAL,
+      :PERCENT_LOWER_I,
+      :PERCENT_LOWER_W,
+      :PERCENT_LOWER_X,
+      :PERCENT_UPPER_I,
+      :PERCENT_UPPER_W,
+      :PIPE,
+      :PIPE_EQUAL,
+      :PIPE_PIPE,
+      :PIPE_PIPE_EQUAL,
+      :PLUS,
+      :PLUS_EQUAL,
+      :QUESTION_MARK,
+      :REGEXP_BEGIN,
+      :REGEXP_END,
+      :SEMICOLON,
+      :SLASH,
+      :SLASH_EQUAL,
+      :STAR,
+      :STAR_EQUAL,
+      :STAR_STAR,
+      :STAR_STAR_EQUAL,
+      :STRING_BEGIN,
+      :STRING_CONTENT,
+      :STRING_END,
+      :SYMBOL_BEGIN,
+      :TILDE,
+      :UAMPERSAND,
+      :UCOLON_COLON,
+      :UDOT_DOT,
+      :UDOT_DOT_DOT,
+      :UMINUS,
+      :UMINUS_NUM,
+      :UPLUS,
+      :USTAR,
+      :USTAR_STAR,
+      :WORDS_SEP,
+      :__END__,
+    ]
+  end
+end