herb 0.11.1 → 0.11.2

data/vendor/prism/templates/src/{token_type.c.erb → tokens.c.erb}

@@ -1,12 +1,12 @@
-#include <string.h>
-
 #include "prism/ast.h"
 
+#include <assert.h>
+
 /**
  * Returns a string representation of the given token type.
  */
-PRISM_EXPORTED_FUNCTION const char *
-pm_token_type_name(pm_token_type_t token_type) {
+const char *
+pm_token_type(pm_token_type_t token_type) {
     switch (token_type) {
 <%- tokens.each do |token| -%>
         case PM_TOKEN_<%= token.name %>:
@@ -27,14 +27,10 @@ pm_token_type_name(pm_token_type_t token_type) {
  * Returns the human name of the given token type.
  */
 const char *
-pm_token_type_human(pm_token_type_t token_type) {
+pm_token_str(pm_token_type_t token_type) {
     switch (token_type) {
         case PM_TOKEN_EOF:
             return "end-of-input";
-        case PM_TOKEN_MISSING:
-            return "missing token";
-        case PM_TOKEN_NOT_PROVIDED:
-            return "not provided token";
         case PM_TOKEN_AMPERSAND:
             return "'&'";
         case PM_TOKEN_AMPERSAND_AMPERSAND:
@@ -57,6 +53,10 @@ pm_token_type_human(pm_token_type_t token_type) {
             return "'!~'";
         case PM_TOKEN_BRACE_LEFT:
             return "'{'";
+        case PM_TOKEN_BRACE_LEFT_ARGUMENT:
+            return "'{'";
+        case PM_TOKEN_BRACE_LEFT_HASH:
+            return "'{'";
         case PM_TOKEN_BRACE_RIGHT:
             return "'}'";
         case PM_TOKEN_BRACKET_LEFT:
@@ -171,6 +171,10 @@ pm_token_type_human(pm_token_type_t token_type) {
             return "'defined?'";
         case PM_TOKEN_KEYWORD_DO:
             return "'do'";
+        case PM_TOKEN_KEYWORD_DO_BLOCK:
+            return "'do'";
+        case PM_TOKEN_KEYWORD_DO_LAMBDA:
+            return "'do'";
         case PM_TOKEN_KEYWORD_DO_LOOP:
             return "'do'";
         case PM_TOKEN_KEYWORD_ELSE:
@@ -271,10 +275,14 @@ pm_token_type_human(pm_token_type_t token_type) {
             return "'->'";
         case PM_TOKEN_NEWLINE:
             return "newline";
+        case PM_TOKEN_NEWLINE_TERMINATOR:
+            return "newline";
         case PM_TOKEN_NUMBERED_REFERENCE:
             return "numbered reference";
         case PM_TOKEN_PARENTHESIS_LEFT:
             return "'('";
+        case PM_TOKEN_PARENTHESIS_LEFT_GROUPING:
+            return "'('";
         case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES:
             return "'('";
         case PM_TOKEN_PARENTHESIS_RIGHT:
@@ -354,7 +362,10 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_USTAR_STAR:
             return "**";
         case PM_TOKEN_WORDS_SEP:
+        case PM_TOKEN_WORDS_SEP_IMPLICIT:
             return "string separator";
+        case PM_TOKEN_XSTRING_BEGIN:
+            return "backtick string literal";
         case PM_TOKEN___END__:
             return "'__END__'";
         case PM_TOKEN_MAXIMUM:
@@ -362,8 +373,8 @@ pm_token_type_human(pm_token_type_t token_type) {
             return "";
     }
 
-    // Provide a default, because some compilers can't determine that the above
-    // switch is exhaustive.
+    /* Provide a default, because some compilers cannot determine that the above
+     * switch is exhaustive. */
     assert(false && "unreachable");
     return "";
 }

data/vendor/prism/templates/template.rb

@@ -6,17 +6,22 @@ require "fileutils"
 require "yaml"
 
 module Prism
-  module Template
+  module Template # :nodoc: all
     SERIALIZE_ONLY_SEMANTICS_FIELDS = ENV.fetch("PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS", false)
-    REMOVE_ON_ERROR_TYPES = SERIALIZE_ONLY_SEMANTICS_FIELDS
     CHECK_FIELD_KIND = ENV.fetch("CHECK_FIELD_KIND", false)
 
-    JAVA_BACKEND = ENV["PRISM_JAVA_BACKEND"] || "truffleruby"
-    JAVA_STRING_TYPE = JAVA_BACKEND == "jruby" ? "org.jruby.RubySymbol" : "String"
+    JAVA_BACKEND = ENV["PRISM_JAVA_BACKEND"] || "default"
+    JAVA_IDENTIFIER_TYPE = JAVA_BACKEND == "truffleruby" ? "String" : "byte[]"
     INCLUDE_NODE_ID = !SERIALIZE_ONLY_SEMANTICS_FIELDS || JAVA_BACKEND == "jruby"
 
     COMMON_FLAGS_COUNT = 2
 
+    # The byte that precedes the child subtrees of a node with a lazy field in
+    # the serialized output (see NodeType#lazy_field). Node types are serialized
+    # as 1 through the number of node types and 0 marks an absent optional node,
+    # so this must be greater than the number of node types.
+    LAZY_MARKER = 0xFF
+
     class Error
       attr_reader :name
 
@@ -49,6 +54,14 @@ module Prism
       end
     end
 
+    # This module contains methods for escaping characters in Doxygen comments.
+    module Doxygen
+      # Similar to /verbatim ... /endverbatim but doesn't wrap the result in a code block.
+      def self.verbatim(value)
+        value.gsub(/[*%!`#<>_+@-]/, '\\\\\0')
+      end
+    end
+
     # A comment attached to a field or node.
     class ConfigComment
       attr_reader :value
@@ -61,6 +74,41 @@ module Prism
         value.each_line { |line| yield line.prepend(" ").rstrip }
       end
 
+      def each_rdoc_line(&block)
+        code_block = false
+        code = +""
+        value.each_line do |line|
+          # Wrap code in explicit language fences so there is syntax highlighting
+          if line.start_with?("    ") && !code_block
+            code_block = true
+            yield " ```rb"
+          elsif !line.strip.empty? && !line.start_with?("    ") && code_block
+            code_block = false
+            rdoc_close_codeblock(code, &block)
+            code = +""
+          end
+
+          if code_block
+            line = line.delete_prefix("    ")
+            code << line
+          end
+          yield line.prepend(" ").rstrip
+        end
+
+        # Close trailing blocks
+        rdoc_close_codeblock(code, &block) if code_block
+      end
+
+      def rdoc_close_codeblock(code, &block)
+        yield " ```"
+        return if code.strip.empty?
+
+        code.gsub!(/^[\s^]*$/, "") # clear marker lines
+        encoded = [code].pack("m0").tr("+/", "-_").delete("=") # base64 urlsafe
+        yield ""
+        yield " [Explore code sample](https://ruby.github.io/prism/playground.html##{encoded})"
+      end
+
       def each_java_line(&block)
         ConfigComment.new(JavaDoc.escape(value)).each_line(&block)
       end
@@ -81,6 +129,10 @@ module Prism
         ConfigComment.new(comment).each_line(&block) if comment
       end
 
+      def each_comment_rdoc_line(&block)
+        ConfigComment.new(comment).each_rdoc_line(&block) if comment
+      end
+
       def each_comment_java_line(&block)
         ConfigComment.new(comment).each_java_line(&block) if comment
       end
@@ -97,6 +149,11 @@ module Prism
     # Some node fields can be specialized if they point to a specific kind of
     # node and not just a generic node.
     class NodeKindField < Field
+      # The C type to use for this field as a function parameter.
+      def c_param
+        "struct #{c_type} *#{name}"
+      end
+
       def initialize(kind:, **options)
         @kind = kind
         super(**options)
@@ -142,27 +199,27 @@ module Prism
         if specific_kind
           specific_kind
         elsif union_kind
-          union_kind.join(" | ")
+          "(#{union_kind.join(" | ")})"
         else
           "Prism::node"
         end
       end
 
-      def rbi_class
+      def call_seq_type
         if specific_kind
-          "Prism::#{specific_kind}"
+          specific_kind
         elsif union_kind
-          "T.any(#{union_kind.map { |kind| "Prism::#{kind}" }.join(", ")})"
+          union_kind.join(" | ")
         else
-          "Prism::Node"
+          "Node"
         end
       end
 
       def check_field_kind
         if union_kind
-          "[#{union_kind.join(', ')}].include?(#{name}.class)"
+          "[#{union_kind.join(', ')}, ErrorRecoveryNode].include?(#{name}.class)"
         else
-          "#{name}.is_a?(#{ruby_type})"
+          "#{name}.is_a?(#{ruby_type}) || #{name}.is_a?(ErrorRecoveryNode)"
         end
       end
     end
@@ -174,27 +231,27 @@ module Prism
         if specific_kind
           "#{specific_kind}?"
         elsif union_kind
-          [*union_kind, "nil"].join(" | ")
+          "(#{union_kind.join(" | ")})?"
         else
           "Prism::node?"
         end
       end
 
-      def rbi_class
+      def call_seq_type
         if specific_kind
-          "T.nilable(Prism::#{specific_kind})"
+          "#{specific_kind} | nil"
         elsif union_kind
-          "T.nilable(T.any(#{union_kind.map { |kind| "Prism::#{kind}" }.join(", ")}))"
+          [*union_kind, "nil"].join(" | ")
         else
-          "T.nilable(Prism::Node)"
+          "Node | nil"
         end
       end
 
       def check_field_kind
         if union_kind
-          "[#{union_kind.join(', ')}, NilClass].include?(#{name}.class)"
+          "[#{union_kind.join(', ')}, ErrorRecoveryNode, NilClass].include?(#{name}.class)"
         else
-          "#{name}.nil? || #{name}.is_a?(#{ruby_type})"
+          "#{name}.nil? || #{name}.is_a?(#{ruby_type}) || #{name}.is_a?(ErrorRecoveryNode)"
         end
       end
     end
@@ -202,23 +259,31 @@ module Prism
     # This represents a field on a node that is a list of nodes. We pass them as
     # references and store them directly on the struct.
     class NodeListField < NodeKindField
-      def rbs_class
+      def c_param
+        "pm_node_list_t #{name}"
+      end
+
+      def element_rbs_class
         if specific_kind
-          "Array[#{specific_kind}]"
+          "#{specific_kind}"
         elsif union_kind
-          "Array[#{union_kind.join(" | ")}]"
+          "#{union_kind.join(" | ")}"
         else
-          "Array[Prism::node]"
+          "Prism::node"
         end
       end
 
-      def rbi_class
+      def rbs_class
+        "Array[#{element_rbs_class}]"
+      end
+
+      def call_seq_type
         if specific_kind
-          "T::Array[Prism::#{specific_kind}]"
+          "Array[#{specific_kind}]"
         elsif union_kind
-          "T::Array[T.any(#{union_kind.map { |kind| "Prism::#{kind}" }.join(", ")})]"
+          "Array[#{union_kind.join(" | ")}]"
         else
-          "T::Array[Prism::Node]"
+          "Array[Node]"
         end
       end
 
@@ -228,9 +293,9 @@ module Prism
 
       def check_field_kind
         if union_kind
-          "#{name}.all? { |n| [#{union_kind.join(', ')}].include?(n.class) }"
+          "#{name}.all? { |n| [#{union_kind.join(', ')}, ErrorRecoveryNode].include?(n.class) }"
         else
-          "#{name}.all? { |n| n.is_a?(#{ruby_type}) }"
+          "#{name}.all? { |n| n.is_a?(#{ruby_type}) || n.is_a?(ErrorRecoveryNode) }"
         end
       end
     end
@@ -238,58 +303,74 @@ module Prism
     # This represents a field on a node that is the ID of a string interned
     # through the parser's constant pool.
     class ConstantField < Field
+      def c_param
+        "pm_constant_id_t #{name}"
+      end
+
       def rbs_class
         "Symbol"
       end
 
-      def rbi_class
+      def call_seq_type
         "Symbol"
       end
 
       def java_type
-        JAVA_STRING_TYPE
+        JAVA_IDENTIFIER_TYPE
       end
     end
 
     # This represents a field on a node that is the ID of a string interned
     # through the parser's constant pool and can be optionally null.
     class OptionalConstantField < Field
+      def c_param
+        "pm_constant_id_t #{name}"
+      end
+
       def rbs_class
         "Symbol?"
       end
 
-      def rbi_class
-        "T.nilable(Symbol)"
+      def call_seq_type
+        "Symbol | nil"
       end
 
       def java_type
-        JAVA_STRING_TYPE
+        JAVA_IDENTIFIER_TYPE
       end
     end
 
     # This represents a field on a node that is a list of IDs that are associated
     # with strings interned through the parser's constant pool.
     class ConstantListField < Field
+      def c_param
+        "pm_constant_id_list_t #{name}"
+      end
+
       def rbs_class
         "Array[Symbol]"
       end
 
-      def rbi_class
-        "T::Array[Symbol]"
+      def call_seq_type
+        "Array[Symbol]"
       end
 
       def java_type
-        "#{JAVA_STRING_TYPE}[]"
+        "#{JAVA_IDENTIFIER_TYPE}[]"
       end
     end
 
     # This represents a field on a node that is a string.
     class StringField < Field
+      def c_param
+        "pm_string_t #{name}"
+      end
+
       def rbs_class
         "String"
       end
 
-      def rbi_class
+      def call_seq_type
         "String"
       end
 
@@ -300,6 +381,10 @@ module Prism
 
     # This represents a field on a node that is a location.
     class LocationField < Field
+      def c_param
+        "pm_location_t #{name}"
+      end
+
       def semantic_field?
         false
       end
@@ -308,8 +393,8 @@ module Prism
         "Location"
       end
 
-      def rbi_class
-        "Prism::Location"
+      def call_seq_type
+        "Location"
       end
 
       def java_type
@@ -319,6 +404,10 @@ module Prism
 
     # This represents a field on a node that is a location that is optional.
     class OptionalLocationField < Field
+      def c_param
+        "pm_location_t #{name}"
+      end
+
       def semantic_field?
         false
       end
@@ -327,8 +416,8 @@ module Prism
         "Location?"
       end
 
-      def rbi_class
-        "T.nilable(Prism::Location)"
+      def call_seq_type
+        "Location | nil"
       end
 
       def java_type
@@ -338,11 +427,15 @@ module Prism
 
     # This represents an integer field.
     class UInt8Field < Field
+      def c_param
+        "uint8_t #{name}"
+      end
+
       def rbs_class
         "Integer"
       end
 
-      def rbi_class
+      def call_seq_type
         "Integer"
       end
 
@@ -353,11 +446,15 @@ module Prism
 
     # This represents an integer field.
     class UInt32Field < Field
+      def c_param
+        "uint32_t #{name}"
+      end
+
       def rbs_class
         "Integer"
       end
 
-      def rbi_class
+      def call_seq_type
         "Integer"
       end
 
@@ -369,11 +466,15 @@ module Prism
     # This represents an arbitrarily-sized integer. When it gets to Ruby it will
     # be an Integer.
     class IntegerField < Field
+      def c_param
+        "pm_integer_t #{name}"
+      end
+
       def rbs_class
         "Integer"
       end
 
-      def rbi_class
+      def call_seq_type
         "Integer"
       end
 
@@ -385,11 +486,15 @@ module Prism
     # This represents a double-precision floating point number. When it gets to
     # Ruby it will be a Float.
     class DoubleField < Field
+      def c_param
+        "double #{name}"
+      end
+
       def rbs_class
         "Float"
       end
 
-      def rbi_class
+      def call_seq_type
         "Float"
       end
 
@@ -432,9 +537,6 @@ module Prism
                 when "pattern expression"
                   # the list of all possible types is too long with 37+ different classes
                   "Node"
-                when Hash
-                  kind = kind.fetch("on error")
-                  REMOVE_ON_ERROR_TYPES ? nil : kind
                 else
                   kind
                 end
@@ -462,6 +564,10 @@ module Prism
         ConfigComment.new(comment).each_line(&block)
       end
 
+      def each_comment_rdoc_line(&block)
+        ConfigComment.new(comment).each_rdoc_line(&block)
+      end
+
       def each_comment_java_line(&block)
         ConfigComment.new(comment).each_java_line(&block)
       end
@@ -470,10 +576,19 @@ module Prism
         @semantic_fields ||= @fields.select(&:semantic_field?)
       end
 
-      # Should emit serialized length of node so implementations can skip
-      # the node to enable lazy parsing.
-      def needs_serialized_length?
-        name == "DefNode"
+      # The fields that hold child nodes. In the serialized output, their
+      # subtrees are written in field order ahead of this node's own record.
+      def child_fields
+        @child_fields ||= fields.grep(NodeKindField)
+      end
+
+      # The field whose subtree loaders can skip and deserialize on demand. In
+      # the serialized output, the node's child subtrees are preceded by
+      # LAZY_MARKER and two uint32 values: the byte length of the subtrees
+      # before this field's subtree, and the byte length of this field's
+      # subtree.
+      def lazy_field
+        fields.find { |field| field.name == "body" } if name == "DefNode"
       end
 
       private
@@ -547,8 +662,7 @@ module Prism
         extension = File.extname(filepath.gsub(".erb", ""))
 
         heading =
-          case extension
-          when ".rb"
+          if extension == ".rb"
             <<~HEADING
             # frozen_string_literal: true
             # :markup: markdown
@@ -561,29 +675,9 @@ module Prism
             ++
             =end
 
-            HEADING
-          when ".rbs"
-            <<~HEADING
-            # This file is generated by the templates/template.rb script and should not be
-            # modified manually. See #{filepath}
-            # if you are looking to modify the template
-
-            HEADING
-          when ".rbi"
-            <<~HEADING
-            # typed: strict
-
-            =begin
-            This file is generated by the templates/template.rb script and should not be
-            modified manually. See #{filepath}
-            if you are looking to modify the template
-            =end
-
             HEADING
           else
             <<~HEADING
-            /* :markup: markdown */
-
             /*----------------------------------------------------------------------------*/
             /* This file is generated by the templates/template.rb script and should not  */
             /* be modified manually. See                                                  */
@@ -607,8 +701,14 @@ module Prism
           end
         end
 
-        FileUtils.mkdir_p(File.dirname(write_to))
-        File.write(write_to, contents)
+        begin
+          FileUtils.mkdir_p(File.dirname(write_to))
+          File.write(write_to, contents)
+        rescue SystemCallError # EACCES, EPERM, EROFS, etc.
+          # Fall back to the current directory
+          FileUtils.mkdir_p(File.dirname(name))
+          File.write(name, contents)
+        end
       end
 
       private
@@ -630,10 +730,19 @@ module Prism
             config = YAML.load_file(File.expand_path("../config.yml", __dir__))
             flags = config.fetch("flags").to_h { |flags| [flags["name"], Flags.new(flags)] }
 
+            nodes = config.fetch("nodes").map { |node| NodeType.new(node, flags) }.sort_by(&:name)
+            raise "Too many node types to reserve LAZY_MARKER" if nodes.length >= LAZY_MARKER
+
+            nodes.each do |node|
+              if (field = node.lazy_field) && !field.is_a?(OptionalNodeField)
+                raise "Lazy field #{node.name}##{field.name} must be an optional node field"
+              end
+            end
+
             {
               errors: config.fetch("errors").map { |name| Error.new(name) },
               warnings: config.fetch("warnings").map { |name| Warning.new(name) },
-              nodes: config.fetch("nodes").map { |node| NodeType.new(node, flags) }.sort_by(&:name),
+              nodes: nodes,
               tokens: config.fetch("tokens").map { |token| Token.new(token) },
               flags: flags.values
             }
@@ -644,13 +753,14 @@ module Prism
     TEMPLATES = [
       "ext/prism/api_node.c",
       "include/prism/ast.h",
-      "include/prism/diagnostic.h",
+      "include/prism/ast.hpp",
+      "include/prism/internal/diagnostic.h",
       "javascript/src/deserialize.js",
       "javascript/src/nodes.js",
       "javascript/src/visitor.js",
-      "java/org/prism/Loader.java",
-      "java/org/prism/Nodes.java",
-      "java/org/prism/AbstractNodeVisitor.java",
+      "java/api/src/main/java-templates/org/ruby_lang/prism/Loader.java",
+      "java/api/src/main/java-templates/org/ruby_lang/prism/Nodes.java",
+      "java/api/src/main/java-templates/org/ruby_lang/prism/AbstractNodeVisitor.java",
       "lib/prism/compiler.rb",
       "lib/prism/dispatcher.rb",
       "lib/prism/dot_visitor.rb",
@@ -662,19 +772,11 @@ module Prism
       "lib/prism/serialize.rb",
       "lib/prism/visitor.rb",
       "src/diagnostic.c",
+      "src/json.c",
       "src/node.c",
       "src/prettyprint.c",
       "src/serialize.c",
-      "src/token_type.c",
-      "rbi/prism/dsl.rbi",
-      "rbi/prism/node.rbi",
-      "rbi/prism/visitor.rbi",
-      "sig/prism.rbs",
-      "sig/prism/dsl.rbs",
-      "sig/prism/mutation_compiler.rbs",
-      "sig/prism/node.rbs",
-      "sig/prism/visitor.rbs",
-      "sig/prism/_private/dot_visitor.rbs"
+      "src/tokens.c"
     ]
   end
 end