prism 0.20.0 → 0.22.0

data/lib/prism/parse_result.rb

@@ -91,7 +91,8 @@ module Prism
   class Location
     # A Source object that is used to determine more information from the given
     # offset and length.
-    protected attr_reader :source
+    attr_reader :source
+    protected :source
 
     # The byte offset from the beginning of the source where this location
     # starts.

data/lib/prism/ripper_compat.rb

@@ -97,6 +97,8 @@ module Prism
         result.errors.each do |error|
           on_parse_error(error.message)
         end
+
+        nil
       else
         result.value.accept(self)
       end
@@ -106,41 +108,91 @@ module Prism
     # Visitor methods
     ############################################################################
 
+    # Visit an ArrayNode node.
+    def visit_array_node(node)
+      elements = visit_elements(node.elements) unless node.elements.empty?
+      bounds(node.location)
+      on_array(elements)
+    end
+
     # Visit a CallNode node.
     def visit_call_node(node)
-      if !node.message.match?(/^[[:alpha:]_]/) && node.opening_loc.nil? && node.arguments&.arguments&.length == 1
-        left = visit(node.receiver)
-        right = visit(node.arguments.arguments.first)
+      if node.variable_call?
+        if node.message.match?(/^[[:alpha:]_]/)
+          bounds(node.message_loc)
+          return on_vcall(on_ident(node.message))
+        end
 
-        bounds(node.location)
-        on_binary(left, node.name, right)
+        raise NotImplementedError, "Non-alpha variable call"
+      end
+
+      if node.opening_loc.nil?
+        left = visit(node.receiver)
+        if node.arguments&.arguments&.length == 1
+          right = visit(node.arguments.arguments.first)
+
+          on_binary(left, node.name, right)
+        elsif !node.arguments || node.arguments.empty?
+          on_unary(node.name, left)
+        else
+          raise NotImplementedError, "More than two arguments for operator"
+        end
       else
-        raise NotImplementedError
+        raise NotImplementedError, "Non-nil opening_loc"
       end
     end
 
     # Visit a FloatNode node.
     def visit_float_node(node)
-      bounds(node.location)
-      on_float(node.slice)
+      visit_number(node) { |text| on_float(text) }
     end
 
     # Visit a ImaginaryNode node.
     def visit_imaginary_node(node)
-      bounds(node.location)
-      on_imaginary(node.slice)
+      visit_number(node) { |text| on_imaginary(text) }
     end
 
     # Visit an IntegerNode node.
     def visit_integer_node(node)
+      visit_number(node) { |text| on_int(text) }
+    end
+
+    # Visit a ParenthesesNode node.
+    def visit_parentheses_node(node)
+      body =
+        if node.body.nil?
+          on_stmts_add(on_stmts_new, on_void_stmt)
+        else
+          visit(node.body)
+        end
+
+      bounds(node.location)
+      on_paren(body)
+    end
+
+    # Visit a ProgramNode node.
+    def visit_program_node(node)
+      statements = visit(node.statements)
+      bounds(node.location)
+      on_program(statements)
+    end
+
+    # Visit a RangeNode node.
+    def visit_range_node(node)
+      left = visit(node.left)
+      right = visit(node.right)
+
       bounds(node.location)
-      on_int(node.slice)
+      if node.exclude_end?
+        on_dot3(left, right)
+      else
+        on_dot2(left, right)
+      end
     end
 
     # Visit a RationalNode node.
     def visit_rational_node(node)
-      bounds(node.location)
-      on_rational(node.slice)
+      visit_number(node) { |text| on_rational(text) }
     end
 
     # Visit a StatementsNode node.
@@ -151,13 +203,6 @@ module Prism
       end
     end
 
-    # Visit a ProgramNode node.
-    def visit_program_node(node)
-      statements = visit(node.statements)
-      bounds(node.location)
-      on_program(statements)
-    end
-
     ############################################################################
     # Entrypoints for subclasses
     ############################################################################
@@ -174,6 +219,32 @@ module Prism
 
     private
 
+    # Visit a list of elements, like the elements of an array or arguments.
+    def visit_elements(elements)
+      bounds(elements.first.location)
+      elements.inject(on_args_new) do |args, element|
+        on_args_add(args, visit(element))
+      end
+    end
+
+    # Visit a node that represents a number. We need to explicitly handle the
+    # unary - operator.
+    def visit_number(node)
+      slice = node.slice
+      location = node.location
+
+      if slice[0] == "-"
+        bounds_values(location.start_line, location.start_column + 1)
+        value = yield slice[1..-1]
+
+        bounds(node.location)
+        on_unary(RUBY_ENGINE == "jruby" ? :- : :-@, value)
+      else
+        bounds(location)
+        yield slice
+      end
+    end
+
     # This method is responsible for updating lineno and column information
     # to reflect the current node.
     #
@@ -184,6 +255,13 @@ module Prism
       @column = location.start_column
     end
 
+    # If we need to do something unusual, we can directly update the line number
+    # and column to reflect the current node.
+    def bounds_values(lineno, column)
+      @lineno = lineno
+      @column = column
+    end
+
     # Lazily initialize the parse result.
     def result
       @result ||= Prism.parse(source)

data/lib/prism/serialize.rb

@@ -27,7 +27,7 @@ module Prism
 
     # The minor version of prism that we are expecting to find in the serialized
     # strings.
-    MINOR_VERSION = 20
+    MINOR_VERSION = 22
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.
@@ -244,6 +244,8 @@ module Prism
         case level
         when 0
           :fatal
+        when 1
+          :argument
         else
           raise "Unknown level: #{level}"
         end

data/lib/prism/translation/parser/compiler.rb

@@ -105,14 +105,18 @@ module Prism
         # { a: 1 }
         #   ^^^^
         def visit_assoc_node(node)
-          if node.value.is_a?(ImplicitNode)
-            builder.pair_label([node.key.slice.chomp(":"), srange(node.key.location)])
-          elsif in_pattern && node.value.nil?
-            if node.key.is_a?(SymbolNode)
-              builder.match_hash_var([node.key.unescaped, srange(node.key.location)])
+          if in_pattern
+            if node.value.is_a?(ImplicitNode)
+              if node.key.is_a?(SymbolNode)
+                builder.match_hash_var([node.key.unescaped, srange(node.key.location)])
+              else
+                builder.match_hash_var_from_str(token(node.key.opening_loc), visit_all(node.key.parts), token(node.key.closing_loc))
+              end
             else
-              builder.match_hash_var_from_str(token(node.key.opening_loc), visit_all(node.key.parts), token(node.key.closing_loc))
+              builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value))
             end
+          elsif node.value.is_a?(ImplicitNode)
+            builder.pair_label([node.key.unescaped, srange(node.key.location)])
           elsif node.operator_loc
             builder.pair(visit(node.key), token(node.operator_loc), visit(node.value))
           elsif node.key.is_a?(SymbolNode) && node.key.opening_loc.nil?
@@ -241,53 +245,51 @@ module Prism
             block = nil
           end
 
+          if node.call_operator_loc.nil?
+            case name
+            when :!
+              return visit_block(builder.not_op(token(node.message_loc), token(node.opening_loc), visit(node.receiver), token(node.closing_loc)), block)
+            when :[]
+              return visit_block(builder.index(visit(node.receiver), token(node.opening_loc), visit_all(arguments), token(node.closing_loc)), block)
+            when :[]=
+              if node.message != "[]=" && node.arguments && block.nil? && !node.safe_navigation?
+                return visit_block(
+                  builder.assign(
+                    builder.index_asgn(
+                      visit(node.receiver),
+                      token(node.opening_loc),
+                      visit_all(node.arguments.arguments[...-1]),
+                      token(node.closing_loc),
+                    ),
+                    srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, ["="]),
+                    visit(node.arguments.arguments.last)
+                  ),
+                  block
+                )
+              end
+            end
+          end
+
+          message_loc = node.message_loc
+          call_operator_loc = node.call_operator_loc
+          call_operator = [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)] if call_operator_loc
+
           visit_block(
-            if name == :!
-              builder.not_op(
-                token(node.message_loc),
-                token(node.opening_loc),
-                visit(node.receiver),
-                token(node.closing_loc)
+            if name.end_with?("=") && !message_loc.slice.end_with?("=") && node.arguments && block.nil?
+              builder.assign(
+                builder.attr_asgn(visit(node.receiver), call_operator, token(message_loc)),
+                srange_find(message_loc.end_offset, node.arguments.location.start_offset, ["="]),
+                visit(node.arguments.arguments.last)
               )
-            elsif name == :[]
-              builder.index(
+            else
+              builder.call_method(
                 visit(node.receiver),
+                call_operator,
+                message_loc ? [node.name, srange(message_loc)] : nil,
                 token(node.opening_loc),
                 visit_all(arguments),
                 token(node.closing_loc)
               )
-            elsif name == :[]= && node.message != "[]=" && node.arguments && block.nil?
-              builder.assign(
-                builder.index_asgn(
-                  visit(node.receiver),
-                  token(node.opening_loc),
-                  visit_all(node.arguments.arguments[...-1]),
-                  token(node.closing_loc),
-                ),
-                srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, ["="]),
-                visit(node.arguments.arguments.last)
-              )
-            else
-              message_loc = node.message_loc
-              call_operator_loc = node.call_operator_loc
-              call_operator = [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)] if call_operator_loc
-
-              if name.end_with?("=") && !message_loc.slice.end_with?("=") && node.arguments && block.nil?
-                builder.assign(
-                  builder.attr_asgn(visit(node.receiver), call_operator, token(message_loc)),
-                  srange_find(message_loc.end_offset, node.arguments.location.start_offset, ["="]),
-                  visit(node.arguments.arguments.last)
-                )
-              else
-                builder.call_method(
-                  visit(node.receiver),
-                  call_operator,
-                  message_loc ? [node.name, srange(message_loc)] : nil,
-                  token(node.opening_loc),
-                  visit_all(arguments),
-                  token(node.closing_loc)
-                )
-              end
             end,
             block
           )
@@ -519,8 +521,6 @@ module Prism
         # def self.foo; end
         # ^^^^^^^^^^^^^^^^^
         def visit_def_node(node)
-          forwarding = find_forwarding(node.parameters)
-
           if node.equal_loc
             if node.receiver
               builder.def_endless_singleton(
@@ -530,7 +530,7 @@ module Prism
                 token(node.name_loc),
                 builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false),
                 token(node.equal_loc),
-                node.body&.accept(copy_compiler(forwarding: forwarding))
+                node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters)))
               )
             else
               builder.def_endless_method(
@@ -538,7 +538,7 @@ module Prism
                 token(node.name_loc),
                 builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false),
                 token(node.equal_loc),
-                node.body&.accept(copy_compiler(forwarding: forwarding))
+                node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters)))
               )
             end
           elsif node.receiver
@@ -548,7 +548,7 @@ module Prism
               token(node.operator_loc),
               token(node.name_loc),
               builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false),
-              node.body&.accept(copy_compiler(forwarding: forwarding)),
+              node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters))),
               token(node.end_keyword_loc)
             )
           else
@@ -556,7 +556,7 @@ module Prism
               token(node.def_keyword_loc),
               token(node.name_loc),
               builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false),
-              node.body&.accept(copy_compiler(forwarding: forwarding)),
+              node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters))),
               token(node.end_keyword_loc)
             )
           end
@@ -614,9 +614,7 @@ module Prism
         # foo => [*, bar, *]
         #        ^^^^^^^^^^^
         def visit_find_pattern_node(node)
-          elements = [*node.requireds]
-          elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode)
-          elements.concat(node.posts)
+          elements = [node.left, *node.requireds, node.right]
 
           if node.constant
             builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.find_pattern(nil, visit_all(elements), nil), token(node.closing_loc))
@@ -993,24 +991,24 @@ module Prism
 
         # -> {}
         def visit_lambda_node(node)
+          parameters = node.parameters
+
           builder.block(
             builder.call_lambda(token(node.operator_loc)),
             [node.opening, srange(node.opening_loc)],
-            if node.parameters
-              if node.parameters.is_a?(NumberedParametersNode)
-                visit(node.parameters)
-              else
-                builder.args(
-                  token(node.parameters.opening_loc),
-                  visit(node.parameters),
-                  token(node.parameters.closing_loc),
-                  false
-                )
-              end
-            else
+            if parameters.nil?
               builder.args(nil, [], nil, false)
+            elsif node.parameters.is_a?(NumberedParametersNode)
+              visit(node.parameters)
+            else
+              builder.args(
+                token(node.parameters.opening_loc),
+                visit(node.parameters),
+                token(node.parameters.closing_loc),
+                false
+              )
             end,
-            node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters&.parameters))),
+            node.body&.accept(copy_compiler(forwarding: parameters.is_a?(NumberedParametersNode) ? [] : find_forwarding(parameters&.parameters))),
             [node.closing, srange(node.closing_loc)]
           )
         end
@@ -1064,12 +1062,22 @@ module Prism
 
         # foo in bar
         # ^^^^^^^^^^
-        def visit_match_predicate_node(node)
-          builder.match_pattern_p(
-            visit(node.value),
-            token(node.operator_loc),
-            within_pattern { |compiler| node.pattern.accept(compiler) }
-          )
+        if RUBY_VERSION >= "3.0"
+          def visit_match_predicate_node(node)
+            builder.match_pattern_p(
+              visit(node.value),
+              token(node.operator_loc),
+              within_pattern { |compiler| node.pattern.accept(compiler) }
+            )
+          end
+        else
+          def visit_match_predicate_node(node)
+            builder.match_pattern(
+              visit(node.value),
+              token(node.operator_loc),
+              within_pattern { |compiler| node.pattern.accept(compiler) }
+            )
+          end
         end
 
         # foo => bar
@@ -1096,7 +1104,7 @@ module Prism
         # case of a syntax error. The parser gem doesn't have such a concept, so
         # we invent our own here.
         def visit_missing_node(node)
-          raise CompilationError, "Cannot compile missing nodes"
+          ::AST::Node.new(:missing, [], location: ::Parser::Source::Map.new(srange(node.location)))
         end
 
         # module Foo; end
@@ -1727,29 +1735,29 @@ module Prism
         # Visit a block node on a call.
         def visit_block(call, block)
           if block
+            parameters = block.parameters
+
             builder.block(
               call,
               token(block.opening_loc),
-              if (parameters = block.parameters)
-                if parameters.is_a?(NumberedParametersNode)
-                  visit(parameters)
-                else
-                  builder.args(
-                    token(parameters.opening_loc),
-                    if procarg0?(parameters.parameters)
-                      parameter = parameters.parameters.requireds.first
-                      [builder.procarg0(visit(parameter))].concat(visit_all(parameters.locals))
-                    else
-                      visit(parameters)
-                    end,
-                    token(parameters.closing_loc),
-                    false
-                  )
-                end
-              else
+              if parameters.nil?
                 builder.args(nil, [], nil, false)
+              elsif parameters.is_a?(NumberedParametersNode)
+                visit(parameters)
+              else
+                builder.args(
+                  token(parameters.opening_loc),
+                  if procarg0?(parameters.parameters)
+                    parameter = parameters.parameters.requireds.first
+                    [builder.procarg0(visit(parameter))].concat(visit_all(parameters.locals))
+                  else
+                    visit(parameters)
+                  end,
+                  token(parameters.closing_loc),
+                  false
+                )
               end,
-              block.body&.accept(copy_compiler(forwarding: find_forwarding(block.parameters&.parameters))),
+              block.body&.accept(copy_compiler(forwarding: parameters.is_a?(NumberedParametersNode) ? [] : find_forwarding(parameters&.parameters))),
               token(block.closing_loc)
             )
           else
@@ -1762,9 +1770,9 @@ module Prism
           children = []
           node.parts.each do |part|
             pushing =
-              if part.is_a?(StringNode) && part.unescaped.count("\n") > 1
-                unescaped = part.unescaped.split("\n")
-                escaped = part.content.split("\n")
+              if part.is_a?(StringNode) && part.unescaped.include?("\n")
+                unescaped = part.unescaped.lines(chomp: true)
+                escaped = part.content.lines(chomp: true)
 
                 escaped_lengths =
                   if node.opening.end_with?("'")
@@ -1779,7 +1787,6 @@ module Prism
                 unescaped.zip(escaped_lengths).map do |unescaped_line, escaped_length|
                   end_offset = start_offset + (escaped_length || 0)
                   inner_part = builder.string_internal(["#{unescaped_line}\n", srange_offsets(start_offset, end_offset)])
-
                   start_offset = end_offset
                   inner_part
                 end

data/lib/prism/translation/parser.rb

@@ -26,7 +26,7 @@ module Prism
       Racc_debug_parser = false # :nodoc:
 
       def version # :nodoc:
-        33
+        34
       end
 
       # The default encoding for Ruby files is UTF-8.
@@ -42,9 +42,10 @@ module Prism
         @source_buffer = source_buffer
         source = source_buffer.source
 
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name))
+        offset_cache = build_offset_cache(source)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name), offset_cache)
 
-        build_ast(result.value, build_offset_cache(source))
+        build_ast(result.value, offset_cache)
       ensure
         @source_buffer = nil
       end
@@ -55,7 +56,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name))
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name), offset_cache)
 
         [
           build_ast(result.value, offset_cache),
@@ -67,17 +68,23 @@ module Prism
 
       # Parses a source buffer and returns the AST, the source code comments,
       # and the tokens emitted by the lexer.
-      def tokenize(source_buffer, _recover = false)
+      def tokenize(source_buffer, recover = false)
         @source_buffer = source_buffer
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse_lex(source, filepath: source_buffer.name))
+        result =
+          begin
+            unwrap(Prism.parse_lex(source, filepath: source_buffer.name), offset_cache)
+          rescue ::Parser::SyntaxError
+            raise if !recover
+          end
 
         program, tokens = result.value
+        ast = build_ast(program, offset_cache) if result.success?
 
         [
-          build_ast(program, offset_cache),
+          ast,
           build_comments(result.comments, offset_cache),
           build_tokens(tokens, offset_cache)
         ]
@@ -93,16 +100,23 @@ module Prism
 
       private
 
+      # This is a hook to allow consumers to disable some errors if they don't
+      # want them to block creating the syntax tree.
+      def valid_error?(error)
+        true
+      end
+
       # If there was a error generated during the parse, then raise an
       # appropriate syntax error. Otherwise return the result.
-      def unwrap(result)
-        return result if result.success?
+      def unwrap(result, offset_cache)
+        result.errors.each do |error|
+          next unless valid_error?(error)
 
-        error = result.errors.first
-        offset_cache = build_offset_cache(source_buffer.source)
+          location = build_range(error.location, offset_cache)
+          diagnostics.process(Diagnostic.new(error.message, location))
+        end
 
-        diagnostic = Diagnostic.new(error.message, build_range(error.location, offset_cache))
-        raise ::Parser::SyntaxError, diagnostic
+        result
       end
 
       # Prism deals with offsets in bytes, while the parser gem deals with

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "0.20.0"
+  spec.version = "0.22.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/ruby/prism"
   spec.license = "MIT"
 
-  spec.required_ruby_version = ">= 3.0.0"
+  spec.required_ruby_version = ">= 2.7.0"
 
   spec.require_paths = ["lib"]
   spec.files = [

data/src/diagnostic.c

@@ -63,7 +63,8 @@ typedef struct {
  *
  * For errors, they are:
  *
- * * `PM_ERROR_LEVEL_FATAL` - The level for all errors.
+ * * `PM_ERROR_LEVEL_FATAL` - The default level for errors.
+ * * `PM_ERROR_LEVEL_ARGUMENT` - Errors that should raise ArgumentError.
  *
  * For warnings, they are:
  *
@@ -71,9 +72,13 @@ typedef struct {
  * * `PM_WARNING_LEVEL_VERBOSE` - Warnings that appear with `-w`, as in `ruby -w -c -e 'code'`.
  */
 static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = {
+    // Special error that can be replaced
     [PM_ERR_CANNOT_PARSE_EXPRESSION]            = { "cannot parse the expression", PM_ERROR_LEVEL_FATAL },
 
-    // Errors
+    // Errors that should raise argument errors
+    [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT]     = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_ARGUMENT },
+
+    // Errors that should raise syntax errors
     [PM_ERR_ALIAS_ARGUMENT]                     = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_AMPAMPEQ_MULTI_ASSIGN]              = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_ARGUMENT_AFTER_BLOCK]               = { "unexpected argument after a block argument", PM_ERROR_LEVEL_FATAL },
@@ -187,15 +192,16 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = {
     [PM_ERR_INCOMPLETE_QUESTION_MARK]           = { "incomplete expression at `?`", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INCOMPLETE_VARIABLE_CLASS]          = { "incomplete class variable", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE]       = { "incomplete instance variable", PM_ERROR_LEVEL_FATAL },
-    [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT]     = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_FLOAT_EXPONENT]             = { "invalid exponent", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_BINARY]              = { "invalid binary number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_DECIMAL]             = { "invalid decimal number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_HEXADECIMAL]         = { "invalid hexadecimal number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_OCTAL]               = { "invalid octal number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_UNDERSCORE]          = { "invalid underscore placement in number", PM_ERROR_LEVEL_FATAL },
+    [PM_ERR_INVALID_CHARACTER]                  = { "invalid character 0x%X", PM_ERROR_LEVEL_FATAL },
+    [PM_ERR_INVALID_MULTIBYTE_CHARACTER]        = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_FATAL },
+    [PM_ERR_INVALID_PRINTABLE_CHARACTER]        = { "invalid character `%c`", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_PERCENT]                    = { "invalid `%` token", PM_ERROR_LEVEL_FATAL }, // TODO WHAT?
-    [PM_ERR_INVALID_TOKEN]                      = { "invalid token", PM_ERROR_LEVEL_FATAL }, // TODO WHAT?
     [PM_ERR_INVALID_VARIABLE_GLOBAL]            = { "invalid global variable", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_IT_NOT_ALLOWED]                     = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_LAMBDA_OPEN]                        = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_FATAL },

data/src/encoding.c

@@ -2252,7 +2252,7 @@ static const uint8_t pm_utf_8_dfa[] = {
  */
 static pm_unicode_codepoint_t
 pm_utf_8_codepoint(const uint8_t *b, ptrdiff_t n, size_t *width) {
-    assert(n >= 1);
+    assert(n >= 0);
     size_t maximum = (size_t) n;
 
     uint32_t codepoint;