herb 0.11.1 → 0.11.2

data/vendor/prism/templates/include/prism/ast.hpp.erb

@@ -0,0 +1,31 @@
+/**
+ * @file ast.hpp
+ *
+ * The mapping between node structs and node types, generated from the same
+ * configuration as ast.h so that it covers every node.
+ */
+#ifndef PRISM_AST_HPP
+#define PRISM_AST_HPP
+
+#include "prism/compiler/cplusplus.hpp"
+
+#include "prism/ast.h"
+#include "prism/node_type.hpp"
+
+#include <cstddef>
+#include <type_traits>
+
+namespace prism {
+
+/* Each node struct is cast to and from pm_node_t, which C++ allows only
+ * because the struct is standard-layout with pm_node_t as its first member. */
+<%- nodes.each do |node| -%>
+
+/** Maps pm_<%= node.human %>_t to <%= node.type %>. */
+template <> struct node_type<pm_<%= node.human %>_t> : std::integral_constant<pm_node_type_t, <%= node.type %>> {};
+static_assert(std::is_standard_layout_v<pm_<%= node.human %>_t> && offsetof(pm_<%= node.human %>_t, base) == 0, "pm_<%= node.human %>_t must begin with its pm_node_t");
+<%- end -%>
+
+}
+
+#endif

data/vendor/prism/templates/include/prism/internal/diagnostic.h.erb

@@ -0,0 +1,60 @@
+#ifndef PRISM_INTERNAL_DIAGNOSTIC_H
+#define PRISM_INTERNAL_DIAGNOSTIC_H
+
+#include "prism/internal/list.h"
+
+#include "prism/arena.h"
+#include "prism/diagnostic.h"
+
+/*
+ * The diagnostic IDs of all of the diagnostics, used to communicate the types
+ * of errors between the parser and the user.
+ */
+typedef enum {
+    /* These are the error diagnostics. */
+    <%- errors.each do |error| -%>
+    PM_ERR_<%= error.name %>,
+    <%- end -%>
+
+    /* These are the warning diagnostics. */
+    <%- warnings.each do |warning| -%>
+    PM_WARN_<%= warning.name %>,
+    <%- end -%>
+} pm_diagnostic_id_t;
+
+/*
+ * This struct represents a diagnostic generated during parsing.
+ */
+struct pm_diagnostic_t {
+    /* The embedded base node. */
+    pm_list_node_t node;
+
+    /* The location of the diagnostic in the source. */
+    pm_location_t location;
+
+    /* The ID of the diagnostic. */
+    pm_diagnostic_id_t diag_id;
+
+    /* The message associated with the diagnostic. */
+    const char *message;
+
+    /*
+     * The level of the diagnostic, see `pm_error_level_t` and
+     * `pm_warning_level_t` for possible values.
+     */
+    uint8_t level;
+};
+
+/*
+ * Append a diagnostic to the given list of diagnostics that is using shared
+ * memory for its message.
+ */
+void pm_diagnostic_list_append(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id);
+
+/*
+ * Append a diagnostic to the given list of diagnostics that is using a format
+ * string for its message.
+ */
+void pm_diagnostic_list_append_format(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...);
+
+#endif

data/vendor/prism/templates/java/{org → api/src/main/java-templates/org/ruby_lang}/prism/AbstractNodeVisitor.java.erb

@@ -1,4 +1,4 @@
-package org.prism;
+package org.ruby_lang.prism;
 
 // GENERATED BY <%= File.basename(__FILE__) %>
 // @formatter:off

data/vendor/prism/templates/java/{org → api/src/main/java-templates/org/ruby_lang}/prism/Loader.java.erb

@@ -1,77 +1,67 @@
-<%- string_type = Prism::Template::JAVA_STRING_TYPE -%>
-package org.prism;
-
-import org.prism.Nodes;
+<%- id_type = Prism::Template::JAVA_IDENTIFIER_TYPE -%>
+package org.ruby_lang.prism;
 
 import java.lang.Short;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.util.Locale;
 
 // GENERATED BY <%= File.basename(__FILE__) %>
 // @formatter:off
 public class Loader {
 
-    public static ParseResult load(byte[] serialized, byte[] sourceBytes) {
-        return new Loader(serialized, sourceBytes).load();
+    public static ParseResult load(byte[] serialized) {
+        return new Loader(serialized).load();
     }
 
     // Overridable methods
 
-    public Charset getEncodingCharset(String encodingName) {
-        encodingName = encodingName.toLowerCase(Locale.ROOT);
-        if (encodingName.equals("ascii-8bit")) {
-            return StandardCharsets.US_ASCII;
-        }
-        return Charset.forName(encodingName);
-    }
-
-    public <%= string_type %> bytesToName(byte[] bytes) {
-        <%- if string_type == "String" -%>
-        return new String(bytes, encodingCharset).intern();
+    public <%= id_type %> bytesToName(byte[] serialized, int offset, int length) {
+        <%- if id_type == "byte[]" -%>
+        byte[] bytes = new byte[length];
+        System.arraycopy(serialized, offset, bytes, 0, length);
+        return bytes;
         <%- else -%>
-        return null; // Must be implemented by subclassing Loader
+        throw new AbstractMethodError("Loader.bytesToName(<%= id_type %>) is not implemented");
         <%- end -%>
     }
 
+    /**
+     * Whether to skip the bodies of DefNodes while loading. A lazily loaded
+     * DefNode has a null body until it is replaced with DefNode#getNonLazy(),
+     * which loads the whole DefNode again from the serialized output.
+     */
+    protected boolean loadDefNodesLazily() {
+        return false;
+    }
+
     private static final class ConstantPool {
 
         private final Loader loader;
-        private final byte[] source;
         private final int bufferOffset;
-        private final <%= string_type %>[] cache;
+        private final <%= id_type %>[] cache;
 
-        ConstantPool(Loader loader, byte[] source, int bufferOffset, int length) {
+        ConstantPool(Loader loader, int bufferOffset, int length) {
             this.loader = loader;
-            this.source = source;
             this.bufferOffset = bufferOffset;
-            cache = new <%= string_type %>[length];
+            <%- if id_type == "String" -%>
+            this.cache = new <%= id_type %>[length];
+            <%- else -%>
+            this.cache = new byte[length][];
+            <%- end -%>
         }
 
-        <%= string_type %> get(ByteBuffer buffer, int oneBasedIndex) {
+        <%= id_type %> get(ByteBuffer buffer, int oneBasedIndex) {
             int index = oneBasedIndex - 1;
-            <%= string_type %> constant = cache[index];
+            <%= id_type %> constant = cache[index];
 
             if (constant == null) {
                 int offset = bufferOffset + index * 8;
                 int start = buffer.getInt(offset);
                 int length = buffer.getInt(offset + 4);
 
-                byte[] bytes = new byte[length];
-
-                if (Integer.compareUnsigned(start, 0x7FFFFFFF) <= 0) {
-                    System.arraycopy(source, start, bytes, 0, length);
-                } else {
-                    int position = buffer.position();
-                    buffer.position(start & 0x7FFFFFFF);
-                    buffer.get(bytes, 0, length);
-                    buffer.position(position);
-                }
-
-                constant = loader.bytesToName(bytes);
+                constant = loader.bytesToName(loader.serialized, start, length);
                 cache[index] = constant;
             }
 
@@ -80,28 +70,28 @@ public class Loader {
 
     }
 
+    private final byte[] serialized;
     private final ByteBuffer buffer;
-    private final Nodes.Source source;
     protected String encodingName;
-    <%- if string_type == "String" -%>
-    private Charset encodingCharset;
-    <%- end -%>
     private ConstantPool constantPool;
+    private Nodes.Source source = null;
 
-    protected Loader(byte[] serialized, byte[] sourceBytes) {
+    protected Loader(byte[] serialized) {
+        this.serialized = serialized;
         this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder());
-        this.source = new Nodes.Source(sourceBytes);
     }
 
     protected ParseResult load() {
+        this.source = new Nodes.Source();
+
         expect((byte) 'P', "incorrect prism header");
         expect((byte) 'R', "incorrect prism header");
         expect((byte) 'I', "incorrect prism header");
         expect((byte) 'S', "incorrect prism header");
         expect((byte) 'M', "incorrect prism header");
 
-        expect((byte) 1, "prism major version does not match");
-        expect((byte) 9, "prism minor version does not match");
+        expect((byte) 2, "prism major version does not match");
+        expect((byte) 1, "prism minor version does not match");
         expect((byte) 0, "prism patch version does not match");
 
         expect((byte) 1, "Loader.java requires no location fields in the serialized output");
@@ -111,63 +101,49 @@ public class Loader {
         byte[] encodingNameBytes = new byte[encodingLength];
         buffer.get(encodingNameBytes);
         this.encodingName = new String(encodingNameBytes, StandardCharsets.US_ASCII);
-        <%- if string_type == "String" -%>
-        this.encodingCharset = getEncodingCharset(this.encodingName);
-        <%- end -%>
 
         source.setStartLine(loadVarSInt());
+
+        // Skip the 8-byte hash of the source, which this library does not expose.
+        buffer.getLong();
+
         source.setLineOffsets(loadLineOffsets());
 
         ParseResult.MagicComment[] magicComments = loadMagicComments();
         Nodes.Location dataLocation = loadOptionalLocation();
         ParseResult.Error[] errors = loadErrors();
         ParseResult.Warning[] warnings = loadWarnings();
+        boolean continuable = buffer.get() != 0;
 
         int constantPoolBufferOffset = buffer.getInt();
         int constantPoolLength = loadVarUInt();
-        this.constantPool = new ConstantPool(this, source.bytes, constantPoolBufferOffset, constantPoolLength);
+        this.constantPool = new ConstantPool(this, constantPoolBufferOffset, constantPoolLength);
 
         Nodes.Node node;
         if (errors.length == 0) {
-            node = loadNode();
+            node = loadNodes(constantPoolBufferOffset, -1);
 
             int left = constantPoolBufferOffset - buffer.position();
             if (left != 0) {
                 throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left");
             }
 
-            boolean[] newlineMarked = new boolean[1 + source.getLineCount()];
-            MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked);
+            MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source);
             node.accept(visitor);
         } else {
             node = null;
         }
 
-        return new ParseResult(node, magicComments, dataLocation, errors, warnings, source);
+        return new ParseResult(node, magicComments, dataLocation, errors, warnings, continuable, source);
     }
 
-    private byte[] loadEmbeddedString() {
+    private byte[] loadString() {
         int length = loadVarUInt();
         byte[] bytes = new byte[length];
         buffer.get(bytes);
         return bytes;
     }
 
-    private byte[] loadString() {
-        switch (buffer.get()) {
-            case 1:
-                int start = loadVarUInt();
-                int length = loadVarUInt();
-                byte[] bytes = new byte[length];
-                System.arraycopy(source.bytes, start, bytes, 0, length);
-                return bytes;
-            case 2:
-                return loadEmbeddedString();
-            default:
-                throw new Error("Expected 0 or 1 but was " + buffer.get());
-        }
-    }
-
     private int[] loadLineOffsets() {
         int count = loadVarUInt();
         int[] lineOffsets = new int[count];
@@ -199,7 +175,7 @@ public class Loader {
         // error messages only contain ASCII characters
         for (int i = 0; i < count; i++) {
             Nodes.ErrorType type = Nodes.ERROR_TYPES[loadVarUInt()];
-            byte[] bytes = loadEmbeddedString();
+            byte[] bytes = loadString();
             String message = new String(bytes, StandardCharsets.US_ASCII);
             Nodes.Location location = loadLocation();
             ParseResult.ErrorLevel level = ParseResult.ERROR_LEVELS[buffer.get()];
@@ -218,7 +194,7 @@ public class Loader {
         // warning messages only contain ASCII characters
         for (int i = 0; i < count; i++) {
             Nodes.WarningType type = Nodes.WARNING_TYPES[loadVarUInt() - <%= errors.length %>];
-            byte[] bytes = loadEmbeddedString();
+            byte[] bytes = loadString();
             String message = new String(bytes, StandardCharsets.US_ASCII);
             Nodes.Location location = loadLocation();
             ParseResult.WarningLevel level = ParseResult.WARNING_LEVELS[buffer.get()];
@@ -230,20 +206,11 @@ public class Loader {
         return warnings;
     }
 
-    private Nodes.Node loadOptionalNode() {
-        if (buffer.get(buffer.position()) != 0) {
-            return loadNode();
-        } else {
-            buffer.position(buffer.position() + 1); // continue after the 0 byte
-            return null;
-        }
-    }
-
-    private <%= string_type %> loadConstant() {
+    private <%= id_type %> loadConstant() {
         return constantPool.get(buffer, loadVarUInt());
     }
 
-    private <%= string_type %> loadOptionalConstant() {
+    private <%= id_type %> loadOptionalConstant() {
         if (buffer.get(buffer.position()) != 0) {
             return loadConstant();
         } else {
@@ -252,12 +219,16 @@ public class Loader {
         }
     }
 
-    private <%= string_type %>[] loadConstants() {
+    private <%= id_type %>[] loadConstants() {
         int length = loadVarUInt();
         if (length == 0) {
-            return Nodes.EMPTY_STRING_ARRAY;
+            return Nodes.EMPTY_IDENTIFIER_ARRAY;
         }
-        <%= string_type %>[] constants = new <%= string_type %>[length];
+        <%- if id_type == "String" -%>
+        <%= id_type %>[] constants = new <%= id_type %>[length];
+        <%- else -%>
+        <%= id_type %>[] constants = new byte[length][];
+        <%- end -%>
         for (int i = 0; i < length; i++) {
             constants[i] = constantPool.get(buffer, loadVarUInt());
         }
@@ -356,67 +327,195 @@ public class Loader {
         return negative ? result.negate() : result;
     }
 
-    private Nodes.Node loadNode() {
-        int type = buffer.get() & 0xFF;
+    private static final int LAZY_MARKER = <%= Prism::Template::LAZY_MARKER %>;
+
+    /**
+     * The stack of nodes that have been loaded but not yet attached to their
+     * parent, along with the lazy subtrees that are pending.
+     */
+    private static final class NodeStack {
+
+        private Nodes.Node[] nodes = new Nodes.Node[64];
+        private int size = 0;
+
+        /** For each pending lazy subtree, its marker position, start, and end. */
+        private int[] lazy = new int[3 * 8];
+        private int lazySize = 0;
+
+        void push(Nodes.Node node) {
+            if (size == nodes.length) {
+                nodes = java.util.Arrays.copyOf(nodes, size * 2);
+            }
+            nodes[size++] = node;
+        }
+
+        Nodes.Node pop() {
+            Nodes.Node node = nodes[--size];
+            nodes[size] = null;
+            return node;
+        }
+
+        void pushLazy(int marker, int start, int end) {
+            if (lazySize * 3 == lazy.length) {
+                lazy = java.util.Arrays.copyOf(lazy, lazy.length * 2);
+            }
+            lazy[lazySize * 3] = marker;
+            lazy[lazySize * 3 + 1] = start;
+            lazy[lazySize * 3 + 2] = end;
+            lazySize++;
+        }
+
+        /**
+         * Returns the marker position of the most recently pushed lazy
+         * subtree, or -1 if there is none. That subtree always belongs to the
+         * node whose record is being loaded, because the lazy subtrees of its
+         * descendants are popped when their own records are loaded.
+         */
+        int popLazy() {
+            return lazySize == 0 ? -1 : lazy[--lazySize * 3];
+        }
+    }
+
+    /**
+     * Loads the nodes serialized from the current position, stopping at the
+     * given offset once a single root remains, and returns that root. Nodes
+     * are serialized in post-order, so the children of each node are the
+     * topmost values on the stack, in field order, by the time its record is
+     * loaded. The lazy marker at eagerMarker, if any, is loaded eagerly even
+     * when DefNodes are loaded lazily.
+     */
+    private Nodes.Node loadNodes(int end, int eagerMarker) {
+        boolean lazy = loadDefNodesLazily();
+        NodeStack stack = new NodeStack();
+
+        while (buffer.position() < end || stack.size != 1) {
+            if (stack.lazySize > 0 && buffer.position() == stack.lazy[stack.lazySize * 3 - 2]) {
+                buffer.position(stack.lazy[stack.lazySize * 3 - 1]);
+                continue;
+            }
+
+            int type = buffer.get() & 0xFF;
+            switch (type) {
+                case 0:
+                    stack.push(null);
+                    break;
+                case LAZY_MARKER: {
+                    int marker = buffer.position() - 1;
+                    int skip = buffer.getInt();
+                    int length = buffer.getInt();
+                    if (lazy && marker != eagerMarker) {
+                        int start = buffer.position() + skip;
+                        stack.pushLazy(marker, start, start + length);
+                    }
+                    break;
+                }
+                <%- nodes.each.with_index(1) do |node, index| -%>
+                case <%= index %>:
+                    stack.push(load<%= node.name %>(stack));
+                    break;
+                <%- end -%>
+                default:
+                    throw new Error("Unknown node type: " + type);
+            }
+        }
+
+        return stack.pop();
+    }
+
+    Nodes.DefNode loadNonLazyDefNode(int lazyPosition) {
+        Nodes.DefNode node;
+        // This method mutates the buffer position and may be called from different threads so we must synchronize
+        synchronized (this) {
+            buffer.position(lazyPosition + 1);
+            int skip = buffer.getInt();
+            int length = buffer.getInt();
+            int end = buffer.position() + skip + length;
+
+            buffer.position(lazyPosition);
+            node = (Nodes.DefNode) loadNodes(end, lazyPosition);
+        }
+
+        MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source);
+        node.accept(visitor);
+
+        return node;
+    }
+    <%- array_types = [] -%>
+    <%- nodes.each do |node| -%>
+
+    private Nodes.<%= node.name %> load<%= node.name %>(NodeStack stack) {
         <%- if Prism::Template::INCLUDE_NODE_ID -%>
         int nodeId = loadVarUInt();
         <%- end -%>
         int startOffset = loadVarUInt();
         int length = loadVarUInt();
-
-        switch (type) {
-            <%- array_types = [] -%>
-            <%- nodes.each_with_index do |node, index| -%>
-            case <%= index + 1 %>:
-            <%-
-            params = []
-            params << "nodeId" if Prism::Template::INCLUDE_NODE_ID
-            params << "startOffset" << "length"
-            params << "buffer.getInt()" if node.needs_serialized_length?
-            params << "loadFlags()" if node.flags
-            params.concat node.semantic_fields.map { |field|
-              case field
-              when Prism::Template::NodeField then "#{field.java_cast}loadNode()"
-              when Prism::Template::OptionalNodeField then "#{field.java_cast}loadOptionalNode()"
-              when Prism::Template::StringField then "loadString()"
-              when Prism::Template::NodeListField then
-                element_type = field.java_type.sub('[]', '')
-                array_types << element_type
-                "load#{element_type}s()"
-              when Prism::Template::ConstantField then "loadConstant()"
-              when Prism::Template::OptionalConstantField then "loadOptionalConstant()"
-              when Prism::Template::ConstantListField then "loadConstants()"
-              when Prism::Template::LocationField then "loadLocation()"
-              when Prism::Template::OptionalLocationField then "loadOptionalLocation()"
-              when Prism::Template::UInt8Field then "buffer.get()"
-              when Prism::Template::UInt32Field then "loadVarUInt()"
-              when Prism::Template::IntegerField then "loadInteger()"
-              when Prism::Template::DoubleField then "buffer.getDouble()"
-              else raise
-              end
-            }
-            -%>
-                return new Nodes.<%= node.name %>(<%= params.join(", ") -%>);
-            <%- end -%>
-            default:
-                throw new Error("Unknown node type: " + type);
-        }
+        <%- if node.flags -%>
+        short flags = loadFlags();
+        <%- end -%>
+        <%- node.semantic_fields.each do |field| -%>
+        <%- case field -%>
+        <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%>
+        <%- when Prism::Template::NodeListField -%>
+        int <%= field.name %>Size = loadVarUInt();
+        <%- else -%>
+        var field_<%= field.name %> = <%=
+          case field
+          when Prism::Template::StringField then "loadString()"
+          when Prism::Template::ConstantField then "loadConstant()"
+          when Prism::Template::OptionalConstantField then "loadOptionalConstant()"
+          when Prism::Template::ConstantListField then "loadConstants()"
+          when Prism::Template::LocationField then "loadLocation()"
+          when Prism::Template::OptionalLocationField then "loadOptionalLocation()"
+          when Prism::Template::UInt8Field then "buffer.get()"
+          when Prism::Template::UInt32Field then "loadVarUInt()"
+          when Prism::Template::IntegerField then "loadInteger()"
+          when Prism::Template::DoubleField then "buffer.getDouble()"
+          else raise
+          end
+        %>;
+        <%- end -%>
+        <%- end -%>
+        <%- if node.lazy_field -%>
+        int lazyPosition = stack.popLazy();
+        <%- end -%>
+        <%- node.child_fields.reverse_each do |field| -%>
+        <%- case field -%>
+        <%- when Prism::Template::NodeListField -%>
+        <%- element_type = field.java_type.sub("[]", ""); array_types << element_type -%>
+        var field_<%= field.name %> = pop<%= element_type %>s(stack, <%= field.name %>Size);
+        <%- else -%>
+        <%- if field == node.lazy_field -%>
+        var field_<%= field.name %> = lazyPosition == -1 ? <%= field.java_cast %>stack.pop() : null;
+        <%- else -%>
+        var field_<%= field.name %> = <%= field.java_cast %>stack.pop();
+        <%- end -%>
+        <%- end -%>
+        <%- end -%>
+        <%-
+          params = []
+          params << "nodeId" if Prism::Template::INCLUDE_NODE_ID
+          params << "startOffset" << "length"
+          params << "lazyPosition" << "lazyPosition == -1 ? null : this" if node.lazy_field
+          params << "flags" if node.flags
+          params.concat(node.semantic_fields.map { |field| "field_#{field.name}" })
+        -%>
+        return new Nodes.<%= node.name %>(<%= params.join(", ") %>);
     }
+    <%- end -%>
     <%- array_types.uniq.each do |type| -%>
 
     private static final Nodes.<%= type %>[] EMPTY_<%= type %>_ARRAY = {};
 
-    private Nodes.<%= type %>[] load<%= type %>s() {
-        int length = loadVarUInt();
+    private static Nodes.<%= type %>[] pop<%= type %>s(NodeStack stack, int length) {
         if (length == 0) {
             return EMPTY_<%= type %>_ARRAY;
         }
         Nodes.<%= type %>[] nodes = new Nodes.<%= type %>[length];
-        for (int i = 0; i < length; i++) {
-            <%- if type == 'Node' -%>
-            nodes[i] = loadNode();
+        for (int i = length - 1; i >= 0; i--) {
+            <%- if type == "Node" -%>
+            nodes[i] = stack.pop();
             <%- else -%>
-            nodes[i] = (Nodes.<%= type %>) loadNode();
+            nodes[i] = (Nodes.<%= type %>) stack.pop();
             <%- end -%>
         }
         return nodes;