yarp 0.10.0 → 0.12.0

data/lib/yarp.rb

@@ -20,6 +20,10 @@ module YARP
       offsets.bsearch_index { |offset| offset > value } || offsets.length
     end
 
+    def line_offset(value)
+      offsets[line(value) - 1]
+    end
+
     def column(value)
       value - offsets[line(value) - 1]
     end
@@ -46,10 +50,14 @@ module YARP
     # The length of this location in bytes.
     attr_reader :length
 
+    # The list of comments attached to this location
+    attr_reader :comments
+
     def initialize(source, start_offset, length)
       @source = source
       @start_offset = start_offset
       @length = length
+      @comments = []
     end
 
     # Create a new location object with the given options.
@@ -81,6 +89,12 @@ module YARP
       source.line(start_offset)
     end
 
+    # The content of the line where this location starts before this location.
+    def start_line_slice
+      offset = source.line_offset(start_offset)
+      source.slice(offset, start_offset - offset)
+    end
+
     # The line number where this location ends.
     def end_line
       source.line(end_offset - 1)
@@ -95,7 +109,7 @@ module YARP
     # The column number in bytes where this location ends from the start of the
     # line.
     def end_column
-      source.column(end_offset - 1)
+      source.column(end_offset)
     end
 
     def deconstruct_keys(keys)
@@ -141,6 +155,15 @@ module YARP
     def deconstruct_keys(keys)
       { type: type, location: location }
     end
+
+    # Returns true if the comment happens on the same line as other code and false if the comment is by itself
+    def trailing?
+      type == :inline && !location.start_line_slice.strip.empty?
+    end
+
+    def inspect
+      "#<YARP::Comment @type=#{@type.inspect} @location=#{@location.inspect}>"
+    end
   end
 
   # This represents an error that was encountered during parsing.
@@ -155,6 +178,10 @@ module YARP
     def deconstruct_keys(keys)
       { message: message, location: location }
     end
+
+    def inspect
+      "#<YARP::ParseError @message=#{@message.inspect} @location=#{@location.inspect}>"
+    end
   end
 
   # This represents a warning that was encountered during parsing.
@@ -169,6 +196,10 @@ module YARP
     def deconstruct_keys(keys)
       { message: message, location: location }
     end
+
+    def inspect
+      "#<YARP::ParseWarning @message=#{@message.inspect} @location=#{@location.inspect}>"
+    end
   end
 
   # A class that knows how to walk down the tree. None of the individual visit
@@ -217,45 +248,6 @@ module YARP
     def failure?
       !success?
     end
-
-    # Keep in sync with Java MarkNewlinesVisitor
-    class MarkNewlinesVisitor < YARP::Visitor
-      def initialize(newline_marked)
-        @newline_marked = newline_marked
-      end
-
-      def visit_block_node(node)
-        old_newline_marked = @newline_marked
-        @newline_marked = Array.new(old_newline_marked.size, false)
-        begin
-          super(node)
-        ensure
-          @newline_marked = old_newline_marked
-        end
-      end
-      alias_method :visit_lambda_node, :visit_block_node
-
-      def visit_if_node(node)
-        node.set_newline_flag(@newline_marked)
-        super(node)
-      end
-      alias_method :visit_unless_node, :visit_if_node
-
-      def visit_statements_node(node)
-        node.body.each do |child|
-          child.set_newline_flag(@newline_marked)
-        end
-        super(node)
-      end
-    end
-    private_constant :MarkNewlinesVisitor
-
-    def mark_newlines
-      newline_marked = Array.new(1 + @source.offsets.size, false)
-      visitor = MarkNewlinesVisitor.new(newline_marked)
-      value.accept(visitor)
-      value
-    end
   end
 
   # This represents a token from the Ruby source.
@@ -315,45 +307,72 @@ module YARP
     end
 
     def pretty_print(q)
-      q.group do
-        q.text(self.class.name.split("::").last)
-        location.pretty_print(q)
-        q.text("[Li:#{location.start_line}]") if newline?
-        q.text("(")
-        q.nest(2) do
-          deconstructed = deconstruct_keys([])
-          deconstructed.delete(:location)
+      q.text(inspect.chomp)
+    end
+  end
 
-          q.breakable("")
-          q.seplist(deconstructed, lambda { q.comma_breakable }, :each_value) { |value| q.pp(value) }
-        end
-        q.breakable("")
-        q.text(")")
+  # This object is responsible for generating the output for the inspect method
+  # implementations of child nodes.
+  class NodeInspector
+    attr_reader :prefix, :output
+
+    def initialize(prefix = "")
+      @prefix = prefix
+      @output = +""
+    end
+
+    # Appends a line to the output with the current prefix.
+    def <<(line)
+      output << "#{prefix}#{line}"
+    end
+
+    # This generates a string that is used as the header of the inspect output
+    # for any given node.
+    def header(node)
+      output = +"@ #{node.class.name.split("::").last} ("
+      output << "location: (#{node.location.start_offset}...#{node.location.end_offset})"
+      output << ", newline: true" if node.newline?
+      output << ")\n"
+      output
+    end
+
+    # Generates a string that represents a list of nodes. It handles properly
+    # using the box drawing characters to make the output look nice.
+    def list(prefix, nodes)
+      output = +"(length: #{nodes.length})\n"
+      last_index = nodes.length - 1
+
+      nodes.each_with_index do |node, index|
+        pointer, preadd = (index == last_index) ? ["└── ", "    "] : ["├── ", "│   "]
+        node_prefix = "#{prefix}#{preadd}"
+        output << node.inspect(NodeInspector.new(node_prefix)).sub(node_prefix, "#{prefix}#{pointer}")
       end
+
+      output
     end
-  end
 
-  class FloatNode < Node
-    def value
-      Float(slice)
+    # Generates a string that represents a location field on a node.
+    def location(value)
+      if value
+        "(#{value.start_offset}...#{value.end_offset}) = #{value.slice.inspect}"
+      else
+        "∅"
+      end
     end
-  end
 
-  class ImaginaryNode < Node
-    def value
-      Complex(0, numeric.value)
+    # Generates a string that represents a child node.
+    def child_node(node, append)
+      node.inspect(child_inspector(append)).delete_prefix(prefix)
     end
-  end
 
-  class IntegerNode < Node
-    def value
-      Integer(slice)
+    # Returns a new inspector that can be used to inspect a child node.
+    def child_inspector(append)
+      NodeInspector.new("#{prefix}#{append}")
     end
-  end
 
-  class RationalNode < Node
-    def value
-      Rational(slice.chomp("r"))
+    # Returns the output as a string.
+    def to_str
+      output
     end
   end
 
@@ -421,10 +440,6 @@ module YARP
             # arguments. We get rid of that here.
             names = names.grep_v(Integer)
 
-            # TODO: We don't support numbered local variables yet, so we get rid
-            # of those here.
-            names = names.grep_v(/^_\d$/)
-
             # For some reason, CRuby occasionally pushes this special local
             # variable when there are splat arguments. We get rid of that here.
             names = names.grep_v(:"#arg_rest")
@@ -464,10 +479,10 @@ module YARP
             sorted = [
               *params.requireds.grep(RequiredParameterNode).map(&:name),
               *params.optionals.map(&:name),
-              *((params.rest.name ? params.rest.name.to_sym : :*) if params.rest && params.rest.operator != ","),
+              *((params.rest.name || :*) if params.rest && params.rest.operator != ","),
               *params.posts.grep(RequiredParameterNode).map(&:name),
-              *params.keywords.reject(&:value).map { |param| param.name.chomp(":").to_sym },
-              *params.keywords.select(&:value).map { |param| param.name.chomp(":").to_sym }
+              *params.keywords.reject(&:value).map(&:name),
+              *params.keywords.select(&:value).map(&:name)
             ]
 
             # TODO: When we get a ... parameter, we should be pushing * and &
@@ -533,9 +548,69 @@ require_relative "yarp/node"
 require_relative "yarp/ripper_compat"
 require_relative "yarp/serialize"
 require_relative "yarp/pack"
+require_relative "yarp/pattern"
+
+require_relative "yarp/parse_result/comments"
+require_relative "yarp/parse_result/newlines"
 
 if RUBY_ENGINE == "ruby" and !ENV["YARP_FFI_BACKEND"]
   require "yarp/yarp"
 else
   require_relative "yarp/ffi"
 end
+
+# Reopening the YARP module after yarp/node is required so that constant
+# reflection APIs will find the constants defined in the node file before these.
+# This block is meant to contain extra APIs we define on YARP nodes that aren't
+# templated and are meant as convenience methods.
+module YARP
+  class FloatNode < Node
+    # Returns the value of the node as a Ruby Float.
+    def value
+      Float(slice)
+    end
+  end
+
+  class ImaginaryNode < Node
+    # Returns the value of the node as a Ruby Complex.
+    def value
+      Complex(0, numeric.value)
+    end
+  end
+
+  class IntegerNode < Node
+    # Returns the value of the node as a Ruby Integer.
+    def value
+      Integer(slice)
+    end
+  end
+
+  class InterpolatedRegularExpressionNode < Node
+    # Returns a numeric value that represents the flags that were used to create
+    # the regular expression.
+    def options
+      o = flags & 0b111
+      o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
+      o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
+      o
+    end
+  end
+
+  class RationalNode < Node
+    # Returns the value of the node as a Ruby Rational.
+    def value
+      Rational(slice.chomp("r"))
+    end
+  end
+
+  class RegularExpressionNode < Node
+    # Returns a numeric value that represents the flags that were used to create
+    # the regular expression.
+    def options
+      o = flags & 0b111
+      o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
+      o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
+      o
+    end
+  end
+end

data/src/diagnostic.c

@@ -1,12 +1,269 @@
 #include "yarp/diagnostic.h"
 
+/*
+  ## Message composition
+
+  When composing an error message, use sentence fragments.
+
+  Try describing the property of the code that caused the error, rather than the rule that is being
+  violated. It may help to use a fragment that completes a sentence beginning, "The parser
+  encountered (a) ...". If appropriate, add a description of the rule violation (or other helpful
+  context) after a semicolon.
+
+  For example:, instead of "Control escape sequence cannot be doubled", prefer:
+
+  > "Invalid control escape sequence; control cannot be repeated"
+
+  In some cases, where the failure is more general or syntax expectations are violated, it may make
+  more sense to use a fragment that completes a sentence beginning, "The parser ...".
+
+  For example:
+
+  > "Expected an expression after `(`"
+  > "Cannot parse the expression"
+
+
+  ## Message style guide
+
+  - Use articles like "a", "an", and "the" when appropriate.
+    - e.g., prefer "Cannot parse the expression" to "Cannot parse expression".
+  - Use the common name for tokens and nodes.
+    - e.g., prefer "keyword splat" to "assoc splat"
+    - e.g., prefer "embedded document" to "embdoc"
+  - Capitalize the initial word of the message.
+  - Use back ticks around token literals
+    - e.g., "Expected a `=>` between the hash key and value"
+  - Do not use `.` or other punctuation at the end of the message.
+  - Do not use contractions like "can't". Prefer "cannot" to "can not".
+  - For tokens that can have multiple meanings, reference the token and its meaning.
+    - e.g., "`*` splat argument" is clearer and more complete than "splat argument" or "`*` argument"
+
+
+  ## Error names (YP_ERR_*)
+
+  - When appropriate, prefer node name to token name.
+    - e.g., prefer "SPLAT" to "STAR" in the context of argument parsing.
+  - Prefer token name to common name.
+    - e.g., prefer "STAR" to "ASTERISK".
+  - Try to order the words in the name from more general to more specific,
+    - e.g., "INVALID_NUMBER_DECIMAL" is better than "DECIMAL_INVALID_NUMBER".
+    - When in doubt, look for similar patterns and name them so that they are grouped when lexically
+      sorted. See YP_ERR_ARGUMENT_NO_FORWARDING_* for an example.
+*/
+
+static const char* const diagnostic_messages[YP_DIAGNOSTIC_ID_LEN] = {
+    [YP_ERR_ALIAS_ARGUMENT]                     = "Invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable",
+    [YP_ERR_AMPAMPEQ_MULTI_ASSIGN]              = "Unexpected `&&=` in a multiple assignment",
+    [YP_ERR_ARGUMENT_AFTER_BLOCK]               = "Unexpected argument after a block argument",
+    [YP_ERR_ARGUMENT_BARE_HASH]                 = "Unexpected bare hash argument",
+    [YP_ERR_ARGUMENT_BLOCK_MULTI]               = "Multiple block arguments; only one block is allowed",
+    [YP_ERR_ARGUMENT_FORMAL_CLASS]              = "Invalid formal argument; formal argument cannot be a class variable",
+    [YP_ERR_ARGUMENT_FORMAL_CONSTANT]           = "Invalid formal argument; formal argument cannot be a constant",
+    [YP_ERR_ARGUMENT_FORMAL_GLOBAL]             = "Invalid formal argument; formal argument cannot be a global variable",
+    [YP_ERR_ARGUMENT_FORMAL_IVAR]               = "Invalid formal argument; formal argument cannot be an instance variable",
+    [YP_ERR_ARGUMENT_NO_FORWARDING_AMP]         = "Unexpected `&` when the parent method is not forwarding",
+    [YP_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES]    = "Unexpected `...` when the parent method is not forwarding",
+    [YP_ERR_ARGUMENT_NO_FORWARDING_STAR]        = "Unexpected `*` when the parent method is not forwarding",
+    [YP_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT]   = "Unexpected `*` splat argument after a `**` keyword splat argument",
+    [YP_ERR_ARGUMENT_SPLAT_AFTER_SPLAT]         = "Unexpected `*` splat argument after a `*` splat argument",
+    [YP_ERR_ARGUMENT_TERM_PAREN]                = "Expected a `)` to close the arguments",
+    [YP_ERR_ARGUMENT_UNEXPECTED_BLOCK]          = "Unexpected `{` after a method call without parenthesis",
+    [YP_ERR_ARRAY_ELEMENT]                      = "Expected an element for the array",
+    [YP_ERR_ARRAY_EXPRESSION]                   = "Expected an expression for the array element",
+    [YP_ERR_ARRAY_EXPRESSION_AFTER_STAR]        = "Expected an expression after `*` in the array",
+    [YP_ERR_ARRAY_SEPARATOR]                    = "Expected a `,` separator for the array elements",
+    [YP_ERR_ARRAY_TERM]                         = "Expected a `]` to close the array",
+    [YP_ERR_BEGIN_LONELY_ELSE]                  = "Unexpected `else` in `begin` block; a `rescue` clause must precede `else`",
+    [YP_ERR_BEGIN_TERM]                         = "Expected an `end` to close the `begin` statement",
+    [YP_ERR_BEGIN_UPCASE_BRACE]                 = "Expected a `{` after `BEGIN`",
+    [YP_ERR_BEGIN_UPCASE_TERM]                  = "Expected a `}` to close the `BEGIN` statement",
+    [YP_ERR_BLOCK_PARAM_LOCAL_VARIABLE]         = "Expected a local variable name in the block parameters",
+    [YP_ERR_BLOCK_PARAM_PIPE_TERM]              = "Expected the block parameters to end with `|`",
+    [YP_ERR_BLOCK_TERM_BRACE]                   = "Expected a block beginning with `{` to end with `}`",
+    [YP_ERR_BLOCK_TERM_END]                     = "Expected a block beginning with `do` to end with `end`",
+    [YP_ERR_CANNOT_PARSE_EXPRESSION]            = "Cannot parse the expression",
+    [YP_ERR_CANNOT_PARSE_STRING_PART]           = "Cannot parse the string part",
+    [YP_ERR_CASE_EXPRESSION_AFTER_CASE]         = "Expected an expression after `case`",
+    [YP_ERR_CASE_EXPRESSION_AFTER_WHEN]         = "Expected an expression after `when`",
+    [YP_ERR_CASE_MISSING_CONDITIONS]            = "Expected a `when` or `in` clause after `case`",
+    [YP_ERR_CASE_TERM]                          = "Expected an `end` to close the `case` statement",
+    [YP_ERR_CLASS_IN_METHOD]                    = "Unexpected class definition in a method body",
+    [YP_ERR_CLASS_NAME]                         = "Expected a constant name after `class`",
+    [YP_ERR_CLASS_SUPERCLASS]                   = "Expected a superclass after `<`",
+    [YP_ERR_CLASS_TERM]                         = "Expected an `end` to close the `class` statement",
+    [YP_ERR_CONDITIONAL_ELSIF_PREDICATE]        = "Expected a predicate expression for the `elsif` statement",
+    [YP_ERR_CONDITIONAL_IF_PREDICATE]           = "Expected a predicate expression for the `if` statement",
+    [YP_ERR_CONDITIONAL_TERM]                   = "Expected an `end` to close the conditional clause",
+    [YP_ERR_CONDITIONAL_TERM_ELSE]              = "Expected an `end` to close the `else` clause",
+    [YP_ERR_CONDITIONAL_UNLESS_PREDICATE]       = "Expected a predicate expression for the `unless` statement",
+    [YP_ERR_CONDITIONAL_UNTIL_PREDICATE]        = "Expected a predicate expression for the `until` statement",
+    [YP_ERR_CONDITIONAL_WHILE_PREDICATE]        = "Expected a predicate expression for the `while` statement",
+    [YP_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = "Expected a constant after the `::` operator",
+    [YP_ERR_DEF_ENDLESS]                        = "Could not parse the endless method body",
+    [YP_ERR_DEF_ENDLESS_SETTER]                 = "Invalid method name; a setter method cannot be defined in an endless method definition",
+    [YP_ERR_DEF_NAME]                           = "Expected a method name",
+    [YP_ERR_DEF_NAME_AFTER_RECEIVER]            = "Expected a method name after the receiver",
+    [YP_ERR_DEF_PARAMS_TERM]                    = "Expected a delimiter to close the parameters",
+    [YP_ERR_DEF_PARAMS_TERM_PAREN]              = "Expected a `)` to close the parameters",
+    [YP_ERR_DEF_RECEIVER]                       = "Expected a receiver for the method definition",
+    [YP_ERR_DEF_RECEIVER_TERM]                  = "Expected a `.` or `::` after the receiver in a method definition",
+    [YP_ERR_DEF_TERM]                           = "Expected an `end` to close the `def` statement",
+    [YP_ERR_DEFINED_EXPRESSION]                 = "Expected an expression after `defined?`",
+    [YP_ERR_EMBDOC_TERM]                        = "Could not find a terminator for the embedded document",
+    [YP_ERR_EMBEXPR_END]                        = "Expected a `}` to close the embedded expression",
+    [YP_ERR_EMBVAR_INVALID]                     = "Invalid embedded variable",
+    [YP_ERR_END_UPCASE_BRACE]                   = "Expected a `{` after `END`",
+    [YP_ERR_END_UPCASE_TERM]                    = "Expected a `}` to close the `END` statement",
+    [YP_ERR_ESCAPE_INVALID_CONTROL]             = "Invalid control escape sequence",
+    [YP_ERR_ESCAPE_INVALID_CONTROL_REPEAT]      = "Invalid control escape sequence; control cannot be repeated",
+    [YP_ERR_ESCAPE_INVALID_HEXADECIMAL]         = "Invalid hexadecimal escape sequence",
+    [YP_ERR_ESCAPE_INVALID_META]                = "Invalid meta escape sequence",
+    [YP_ERR_ESCAPE_INVALID_META_REPEAT]         = "Invalid meta escape sequence; meta cannot be repeated",
+    [YP_ERR_ESCAPE_INVALID_UNICODE]             = "Invalid Unicode escape sequence",
+    [YP_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS]    = "Invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags",
+    [YP_ERR_ESCAPE_INVALID_UNICODE_LITERAL]     = "Invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal",
+    [YP_ERR_ESCAPE_INVALID_UNICODE_LONG]        = "Invalid Unicode escape sequence; maximum length is 6 digits",
+    [YP_ERR_ESCAPE_INVALID_UNICODE_TERM]        = "Invalid Unicode escape sequence; needs closing `}`",
+    [YP_ERR_EXPECT_ARGUMENT]                    = "Expected an argument",
+    [YP_ERR_EXPECT_EOL_AFTER_STATEMENT]         = "Expected a newline or semicolon after the statement",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ]   = "Expected an expression after `&&=`",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = "Expected an expression after `||=`",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_COMMA]      = "Expected an expression after `,`",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_EQUAL]      = "Expected an expression after `=`",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS]  = "Expected an expression after `<<`",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_LPAREN]     = "Expected an expression after `(`",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR]   = "Expected an expression after the operator",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_SPLAT]      = "Expected an expression after `*` splat in an argument",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = "Expected an expression after `**` in a hash",
+    [YP_ERR_EXPECT_EXPRESSION_AFTER_STAR]       = "Expected an expression after `*`",
+    [YP_ERR_EXPECT_IDENT_REQ_PARAMETER]         = "Expected an identifier for the required parameter",
+    [YP_ERR_EXPECT_LPAREN_REQ_PARAMETER]        = "Expected a `(` to start a required parameter",
+    [YP_ERR_EXPECT_RBRACKET]                    = "Expected a matching `]`",
+    [YP_ERR_EXPECT_RPAREN]                      = "Expected a matching `)`",
+    [YP_ERR_EXPECT_RPAREN_AFTER_MULTI]          = "Expected a `)` after multiple assignment",
+    [YP_ERR_EXPECT_RPAREN_REQ_PARAMETER]        = "Expected a `)` to end a required parameter",
+    [YP_ERR_EXPECT_STRING_CONTENT]              = "Expected string content after opening string delimiter",
+    [YP_ERR_EXPECT_WHEN_DELIMITER]              = "Expected a delimiter after the predicates of a `when` clause",
+    [YP_ERR_EXPRESSION_BARE_HASH]               = "Unexpected bare hash in expression",
+    [YP_ERR_FOR_COLLECTION]                     = "Expected a collection after the `in` in a `for` statement",
+    [YP_ERR_FOR_INDEX]                          = "Expected an index after `for`",
+    [YP_ERR_FOR_IN]                             = "Expected an `in` after the index in a `for` statement",
+    [YP_ERR_FOR_TERM]                           = "Expected an `end` to close the `for` loop",
+    [YP_ERR_HASH_EXPRESSION_AFTER_LABEL]        = "Expected an expression after the label in a hash",
+    [YP_ERR_HASH_KEY]                           = "Expected a key in the hash literal",
+    [YP_ERR_HASH_ROCKET]                        = "Expected a `=>` between the hash key and value",
+    [YP_ERR_HASH_TERM]                          = "Expected a `}` to close the hash literal",
+    [YP_ERR_HASH_VALUE]                         = "Expected a value in the hash literal",
+    [YP_ERR_HEREDOC_TERM]                       = "Could not find a terminator for the heredoc",
+    [YP_ERR_INCOMPLETE_QUESTION_MARK]           = "Incomplete expression at `?`",
+    [YP_ERR_INCOMPLETE_VARIABLE_CLASS]          = "Incomplete class variable",
+    [YP_ERR_INCOMPLETE_VARIABLE_INSTANCE]       = "Incomplete instance variable",
+    [YP_ERR_INVALID_ENCODING_MAGIC_COMMENT]     = "Unknown or invalid encoding in the magic comment",
+    [YP_ERR_INVALID_FLOAT_EXPONENT]             = "Invalid exponent",
+    [YP_ERR_INVALID_NUMBER_BINARY]              = "Invalid binary number",
+    [YP_ERR_INVALID_NUMBER_DECIMAL]             = "Invalid decimal number",
+    [YP_ERR_INVALID_NUMBER_HEXADECIMAL]         = "Invalid hexadecimal number",
+    [YP_ERR_INVALID_NUMBER_OCTAL]               = "Invalid octal number",
+    [YP_ERR_INVALID_NUMBER_UNDERSCORE]          = "Invalid underscore placement in number",
+    [YP_ERR_INVALID_PERCENT]                    = "Invalid `%` token", // TODO WHAT?
+    [YP_ERR_INVALID_TOKEN]                      = "Invalid token", // TODO WHAT?
+    [YP_ERR_INVALID_VARIABLE_GLOBAL]            = "Invalid global variable",
+    [YP_ERR_LAMBDA_OPEN]                        = "Expected a `do` keyword or a `{` to open the lambda block",
+    [YP_ERR_LAMBDA_TERM_BRACE]                  = "Expected a lambda block beginning with `{` to end with `}`",
+    [YP_ERR_LAMBDA_TERM_END]                    = "Expected a lambda block beginning with `do` to end with `end`",
+    [YP_ERR_LIST_I_LOWER_ELEMENT]               = "Expected a symbol in a `%i` list",
+    [YP_ERR_LIST_I_LOWER_TERM]                  = "Expected a closing delimiter for the `%i` list",
+    [YP_ERR_LIST_I_UPPER_ELEMENT]               = "Expected a symbol in a `%I` list",
+    [YP_ERR_LIST_I_UPPER_TERM]                  = "Expected a closing delimiter for the `%I` list",
+    [YP_ERR_LIST_W_LOWER_ELEMENT]               = "Expected a string in a `%w` list",
+    [YP_ERR_LIST_W_LOWER_TERM]                  = "Expected a closing delimiter for the `%w` list",
+    [YP_ERR_LIST_W_UPPER_ELEMENT]               = "Expected a string in a `%W` list",
+    [YP_ERR_LIST_W_UPPER_TERM]                  = "Expected a closing delimiter for the `%W` list",
+    [YP_ERR_MALLOC_FAILED]                      = "Failed to allocate memory",
+    [YP_ERR_MODULE_IN_METHOD]                   = "Unexpected module definition in a method body",
+    [YP_ERR_MODULE_NAME]                        = "Expected a constant name after `module`",
+    [YP_ERR_MODULE_TERM]                        = "Expected an `end` to close the `module` statement",
+    [YP_ERR_MULTI_ASSIGN_MULTI_SPLATS]          = "Multiple splats in multiple assignment",
+    [YP_ERR_NOT_EXPRESSION]                     = "Expected an expression after `not`",
+    [YP_ERR_NUMBER_LITERAL_UNDERSCORE]          = "Number literal ending with a `_`",
+    [YP_ERR_NUMBERED_PARAMETER_NOT_ALLOWED]     = "Numbered parameters are not allowed alongside explicit parameters",
+    [YP_ERR_NUMBERED_PARAMETER_OUTER_SCOPE]     = "Numbered parameter is already used in outer scope",
+    [YP_ERR_OPERATOR_MULTI_ASSIGN]              = "Unexpected operator for a multiple assignment",
+    [YP_ERR_OPERATOR_WRITE_BLOCK]               = "Unexpected operator after a call with a block",
+    [YP_ERR_PARAMETER_ASSOC_SPLAT_MULTI]        = "Unexpected multiple `**` splat parameters",
+    [YP_ERR_PARAMETER_BLOCK_MULTI]              = "Multiple block parameters; only one block is allowed",
+    [YP_ERR_PARAMETER_NAME_REPEAT]              = "Repeated parameter name",
+    [YP_ERR_PARAMETER_NO_DEFAULT]               = "Expected a default value for the parameter",
+    [YP_ERR_PARAMETER_NO_DEFAULT_KW]            = "Expected a default value for the keyword parameter",
+    [YP_ERR_PARAMETER_NUMBERED_RESERVED]        = "Token reserved for a numbered parameter",
+    [YP_ERR_PARAMETER_ORDER]                    = "Unexpected parameter order",
+    [YP_ERR_PARAMETER_SPLAT_MULTI]              = "Unexpected multiple `*` splat parameters",
+    [YP_ERR_PARAMETER_STAR]                     = "Unexpected parameter `*`",
+    [YP_ERR_PARAMETER_WILD_LOOSE_COMMA]         = "Unexpected `,` in parameters",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_BRACKET]   = "Expected a pattern expression after the `[` operator",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_COMMA]     = "Expected a pattern expression after `,`",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_HROCKET]   = "Expected a pattern expression after `=>`",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_IN]        = "Expected a pattern expression after the `in` keyword",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_KEY]       = "Expected a pattern expression after the key",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_PAREN]     = "Expected a pattern expression after the `(` operator",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_PIN]       = "Expected a pattern expression after the `^` pin operator",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_PIPE]      = "Expected a pattern expression after the `|` operator",
+    [YP_ERR_PATTERN_EXPRESSION_AFTER_RANGE]     = "Expected a pattern expression after the range operator",
+    [YP_ERR_PATTERN_HASH_KEY]                   = "Expected a key in the hash pattern",
+    [YP_ERR_PATTERN_HASH_KEY_LABEL]             = "Expected a label as the key in the hash pattern", // TODO // THIS // AND // ABOVE // IS WEIRD
+    [YP_ERR_PATTERN_IDENT_AFTER_HROCKET]        = "Expected an identifier after the `=>` operator",
+    [YP_ERR_PATTERN_LABEL_AFTER_COMMA]          = "Expected a label after the `,` in the hash pattern",
+    [YP_ERR_PATTERN_REST]                       = "Unexpected rest pattern",
+    [YP_ERR_PATTERN_TERM_BRACE]                 = "Expected a `}` to close the pattern expression",
+    [YP_ERR_PATTERN_TERM_BRACKET]               = "Expected a `]` to close the pattern expression",
+    [YP_ERR_PATTERN_TERM_PAREN]                 = "Expected a `)` to close the pattern expression",
+    [YP_ERR_PIPEPIPEEQ_MULTI_ASSIGN]            = "Unexpected `||=` in a multiple assignment",
+    [YP_ERR_REGEXP_TERM]                        = "Expected a closing delimiter for the regular expression",
+    [YP_ERR_RESCUE_EXPRESSION]                  = "Expected a rescued expression",
+    [YP_ERR_RESCUE_MODIFIER_VALUE]              = "Expected a value after the `rescue` modifier",
+    [YP_ERR_RESCUE_TERM]                        = "Expected a closing delimiter for the `rescue` clause",
+    [YP_ERR_RESCUE_VARIABLE]                    = "Expected an exception variable after `=>` in a rescue statement",
+    [YP_ERR_RETURN_INVALID]                     = "Invalid `return` in a class or module body",
+    [YP_ERR_STRING_CONCATENATION]               = "Expected a string for concatenation",
+    [YP_ERR_STRING_INTERPOLATED_TERM]           = "Expected a closing delimiter for the interpolated string",
+    [YP_ERR_STRING_LITERAL_TERM]                = "Expected a closing delimiter for the string literal",
+    [YP_ERR_SYMBOL_INVALID]                     = "Invalid symbol", // TODO expected symbol? yarp.c ~9719
+    [YP_ERR_SYMBOL_TERM_DYNAMIC]                = "Expected a closing delimiter for the dynamic symbol",
+    [YP_ERR_SYMBOL_TERM_INTERPOLATED]           = "Expected a closing delimiter for the interpolated symbol",
+    [YP_ERR_TERNARY_COLON]                      = "Expected a `:` after the true expression of a ternary operator",
+    [YP_ERR_TERNARY_EXPRESSION_FALSE]           = "Expected an expression after `:` in the ternary operator",
+    [YP_ERR_TERNARY_EXPRESSION_TRUE]            = "Expected an expression after `?` in the ternary operator",
+    [YP_ERR_UNDEF_ARGUMENT]                     = "Invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument",
+    [YP_ERR_UNARY_RECEIVER_BANG]                = "Expected a receiver for unary `!`",
+    [YP_ERR_UNARY_RECEIVER_MINUS]               = "Expected a receiver for unary `-`",
+    [YP_ERR_UNARY_RECEIVER_PLUS]                = "Expected a receiver for unary `+`",
+    [YP_ERR_UNARY_RECEIVER_TILDE]               = "Expected a receiver for unary `~`",
+    [YP_ERR_UNTIL_TERM]                         = "Expected an `end` to close the `until` statement",
+    [YP_ERR_WHILE_TERM]                         = "Expected an `end` to close the `while` statement",
+    [YP_ERR_WRITE_TARGET_READONLY]              = "Immutable variable as a write target",
+    [YP_ERR_WRITE_TARGET_UNEXPECTED]            = "Unexpected write target",
+    [YP_ERR_XSTRING_TERM]                       = "Expected a closing delimiter for the `%x` or backtick string",
+    [YP_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS]    = "Ambiguous first argument; put parentheses or a space even after `-` operator",
+    [YP_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS]     = "Ambiguous first argument; put parentheses or a space even after `+` operator",
+    [YP_WARN_AMBIGUOUS_PREFIX_STAR]             = "Ambiguous `*` has been interpreted as an argument prefix",
+    [YP_WARN_AMBIGUOUS_SLASH]                   = "Ambiguous `/`; wrap regexp in parentheses or add a space after `/` operator",
+};
+
+static const char*
+yp_diagnostic_message(yp_diagnostic_id_t diag_id) {
+    assert(diag_id < YP_DIAGNOSTIC_ID_LEN);
+    const char *message = diagnostic_messages[diag_id];
+    assert(message);
+    return message;
+}
+
 // Append an error to the given list of diagnostic.
 bool
-yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, const char *message) {
+yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, yp_diagnostic_id_t diag_id) {
     yp_diagnostic_t *diagnostic = (yp_diagnostic_t *) malloc(sizeof(yp_diagnostic_t));
     if (diagnostic == NULL) return false;
 
-    *diagnostic = (yp_diagnostic_t) { .start = start, .end = end, .message = message };
+    *diagnostic = (yp_diagnostic_t) { .start = start, .end = end, .message = yp_diagnostic_message(diag_id) };
     yp_list_append(list, (yp_list_node_t *) diagnostic);
     return true;
 }

data/src/enc/yp_unicode.c

@@ -10,7 +10,7 @@ typedef uint32_t yp_unicode_codepoint_t;
 // this table is different from other encodings where we used a lookup table
 // because the indices of those tables are the byte representations, not the
 // codepoints themselves.
-uint8_t yp_encoding_unicode_table[256] = {
+const uint8_t yp_encoding_unicode_table[256] = {
 //  0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x
@@ -31,7 +31,7 @@ uint8_t yp_encoding_unicode_table[256] = {
 };
 
 #define UNICODE_ALPHA_CODEPOINTS_LENGTH 1450
-static yp_unicode_codepoint_t unicode_alpha_codepoints[UNICODE_ALPHA_CODEPOINTS_LENGTH] = {
+static const yp_unicode_codepoint_t unicode_alpha_codepoints[UNICODE_ALPHA_CODEPOINTS_LENGTH] = {
     0x100, 0x2C1,
     0x2C6, 0x2D1,
     0x2E0, 0x2E4,
@@ -760,7 +760,7 @@ static yp_unicode_codepoint_t unicode_alpha_codepoints[UNICODE_ALPHA_CODEPOINTS_
 };
 
 #define UNICODE_ALNUM_CODEPOINTS_LENGTH 1528
-static yp_unicode_codepoint_t unicode_alnum_codepoints[UNICODE_ALNUM_CODEPOINTS_LENGTH] = {
+static const yp_unicode_codepoint_t unicode_alnum_codepoints[UNICODE_ALNUM_CODEPOINTS_LENGTH] = {
     0x100, 0x2C1,
     0x2C6, 0x2D1,
     0x2E0, 0x2E4,
@@ -1528,7 +1528,7 @@ static yp_unicode_codepoint_t unicode_alnum_codepoints[UNICODE_ALNUM_CODEPOINTS_
 };
 
 #define UNICODE_ISUPPER_CODEPOINTS_LENGTH 1296
-static yp_unicode_codepoint_t unicode_isupper_codepoints[UNICODE_ISUPPER_CODEPOINTS_LENGTH] = {
+static const yp_unicode_codepoint_t unicode_isupper_codepoints[UNICODE_ISUPPER_CODEPOINTS_LENGTH] = {
     0x100, 0x100,
     0x102, 0x102,
     0x104, 0x104,
@@ -2180,7 +2180,7 @@ static yp_unicode_codepoint_t unicode_isupper_codepoints[UNICODE_ISUPPER_CODEPOI
 };
 
 static bool
-yp_unicode_codepoint_match(yp_unicode_codepoint_t codepoint, yp_unicode_codepoint_t *codepoints, size_t size) {
+yp_unicode_codepoint_match(yp_unicode_codepoint_t codepoint, const yp_unicode_codepoint_t *codepoints, size_t size) {
     size_t start = 0;
     size_t end = size;