herb 0.7.2-aarch64-linux-musl → 0.7.3-aarch64-linux-musl

data/vendor/prism/src/util/pm_strpbrk.c

@@ -0,0 +1,206 @@
+#include "prism/util/pm_strpbrk.h"
+
+/**
+ * Add an invalid multibyte character error to the parser.
+ */
+static inline void
+pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
+    pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
+}
+
+/**
+ * Set the explicit encoding for the parser to the current encoding.
+ */
+static inline void
+pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) {
+    if (parser->explicit_encoding != NULL) {
+        if (parser->explicit_encoding == parser->encoding) {
+            // Okay, we already locked to this encoding.
+        } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
+            // Not okay, we already found a Unicode escape sequence and this
+            // conflicts.
+            pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name);
+        } else {
+            // Should not be anything else.
+            assert(false && "unreachable");
+        }
+    }
+
+    parser->explicit_encoding = parser->encoding;
+}
+
+/**
+ * This is the default path.
+ */
+static inline const uint8_t *
+pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (source[index] < 0x80) {
+            index++;
+        } else {
+            size_t width = pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index));
+
+            if (width > 0) {
+                index += width;
+            } else if (!validate) {
+                index++;
+            } else {
+                // At this point we know we have an invalid multibyte character.
+                // We'll walk forward as far as we can until we find the next
+                // valid character so that we don't spam the user with a ton of
+                // the same kind of error.
+                const size_t start = index;
+
+                do {
+                    index++;
+                } while (index < maximum && pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+
+                pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * This is the path when the encoding is ASCII-8BIT.
+ */
+static inline const uint8_t *
+pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1);
+        index++;
+    }
+
+    return NULL;
+}
+
+/**
+ * This is the slow path that does care about the encoding.
+ */
+static inline const uint8_t *
+pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (source[index] < 0x80) {
+            index++;
+        } else {
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width);
+
+            if (width > 0) {
+                index += width;
+            } else if (!validate) {
+                index++;
+            } else {
+                // At this point we know we have an invalid multibyte character.
+                // We'll walk forward as far as we can until we find the next
+                // valid character so that we don't spam the user with a ton of
+                // the same kind of error.
+                const size_t start = index;
+
+                do {
+                    index++;
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+
+                pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * This is the fast path that does not care about the encoding because we know
+ * the encoding only supports single-byte characters.
+ */
+static inline const uint8_t *
+pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
+    size_t index = 0;
+    const pm_encoding_t *encoding = parser->encoding;
+
+    while (index < maximum) {
+        if (strchr((const char *) charset, source[index]) != NULL) {
+            return source + index;
+        }
+
+        if (source[index] < 0x80 || !validate) {
+            index++;
+        } else {
+            size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
+            pm_strpbrk_explicit_encoding_set(parser, source, width);
+
+            if (width > 0) {
+                index += width;
+            } else {
+                // At this point we know we have an invalid multibyte character.
+                // We'll walk forward as far as we can until we find the next
+                // valid character so that we don't spam the user with a ton of
+                // the same kind of error.
+                const size_t start = index;
+
+                do {
+                    index++;
+                } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
+
+                pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * Here we have rolled our own version of strpbrk. The standard library strpbrk
+ * has undefined behavior when the source string is not null-terminated. We want
+ * to support strings that are not null-terminated because pm_parse does not
+ * have the contract that the string is null-terminated. (This is desirable
+ * because it means the extension can call pm_parse with the result of a call to
+ * mmap).
+ *
+ * The standard library strpbrk also does not support passing a maximum length
+ * to search. We want to support this for the reason mentioned above, but we
+ * also don't want it to stop on null bytes. Ruby actually allows null bytes
+ * within strings, comments, regular expressions, etc. So we need to be able to
+ * skip past them.
+ *
+ * Finally, we want to support encodings wherein the charset could contain
+ * characters that are trailing bytes of multi-byte characters. For example, in
+ * Shift_JIS, the backslash character can be a trailing byte. In that case we
+ * need to take a slower path and iterate one multi-byte character at a time.
+ */
+const uint8_t *
+pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate) {
+    if (length <= 0) {
+        return NULL;
+    } else if (!parser->encoding_changed) {
+        return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
+    } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
+        return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate);
+    } else if (parser->encoding->multibyte) {
+        return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
+    } else {
+        return pm_strpbrk_single_byte(parser, source, charset, (size_t) length, validate);
+    }
+}

data/vendor/prism/templates/ext/prism/api_node.c.erb

@@ -0,0 +1,282 @@
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+#include "prism/extension.h"
+
+extern VALUE rb_cPrism;
+extern VALUE rb_cPrismNode;
+extern VALUE rb_cPrismSource;
+extern VALUE rb_cPrismToken;
+extern VALUE rb_cPrismLocation;
+
+<%- nodes.each do |node| -%>
+static VALUE rb_cPrism<%= node.name %>;
+<%- end -%>
+
+static VALUE
+pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source, bool freeze) {
+    if (freeze) {
+        VALUE location_argv[] = {
+            source,
+            LONG2FIX(start - parser->start),
+            LONG2FIX(end - start)
+        };
+
+        return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation));
+    } else {
+        uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start)));
+        return ULL2NUM(value);
+    }
+}
+
+VALUE
+pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) {
+    ID type = rb_intern(pm_token_type_name(token->type));
+    VALUE location = pm_location_new(parser, token->start, token->end, source, freeze);
+
+    VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding);
+    if (freeze) rb_obj_freeze(slice);
+
+    VALUE argv[] = { source, ID2SYM(type), slice, location };
+    VALUE value = rb_class_new_instance(4, argv, rb_cPrismToken);
+    if (freeze) rb_obj_freeze(value);
+
+    return value;
+}
+
+static VALUE
+pm_string_new(const pm_string_t *string, rb_encoding *encoding) {
+    return rb_obj_freeze(rb_enc_str_new((const char *) pm_string_source(string), pm_string_length(string), encoding));
+}
+
+VALUE
+pm_integer_new(const pm_integer_t *integer) {
+    VALUE result;
+    if (integer->values == NULL) {
+        result = UINT2NUM(integer->value);
+    } else {
+        VALUE string = rb_str_new(NULL, integer->length * 8);
+        unsigned char *bytes = (unsigned char *) RSTRING_PTR(string);
+
+        size_t offset = integer->length * 8;
+        for (size_t value_index = 0; value_index < integer->length; value_index++) {
+            uint32_t value = integer->values[value_index];
+
+            for (int index = 0; index < 8; index++) {
+                int byte = (value >> (4 * index)) & 0xf;
+                bytes[--offset] = byte < 10 ? byte + '0' : byte - 10 + 'a';
+            }
+        }
+
+        result = rb_funcall(string, rb_intern("to_i"), 1, UINT2NUM(16));
+    }
+
+    if (integer->negative) {
+        result = rb_funcall(result, rb_intern("-@"), 0);
+    }
+
+    return result;
+}
+
+// Create a Prism::Source object from the given parser, after pm_parse() was called.
+VALUE
+pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze) {
+    VALUE source_string = rb_enc_str_new((const char *) parser->start, parser->end - parser->start, encoding);
+
+    VALUE offsets = rb_ary_new_capa(parser->newline_list.size);
+    for (size_t index = 0; index < parser->newline_list.size; index++) {
+        rb_ary_push(offsets, ULONG2NUM(parser->newline_list.offsets[index]));
+    }
+
+    if (freeze) {
+        rb_obj_freeze(source_string);
+        rb_obj_freeze(offsets);
+    }
+
+    VALUE source = rb_funcall(rb_cPrismSource, rb_intern("for"), 3, source_string, LONG2NUM(parser->start_line), offsets);
+    if (freeze) rb_obj_freeze(source);
+
+    return source;
+}
+
+typedef struct pm_node_stack_node {
+    struct pm_node_stack_node *prev;
+    const pm_node_t *visit;
+    bool visited;
+} pm_node_stack_node_t;
+
+static void
+pm_node_stack_push(pm_node_stack_node_t **stack, const pm_node_t *visit) {
+    pm_node_stack_node_t *node = xmalloc(sizeof(pm_node_stack_node_t));
+    node->prev = *stack;
+    node->visit = visit;
+    node->visited = false;
+    *stack = node;
+}
+
+static const pm_node_t *
+pm_node_stack_pop(pm_node_stack_node_t **stack) {
+    pm_node_stack_node_t *current = *stack;
+    const pm_node_t *visit = current->visit;
+
+    *stack = current->prev;
+    xfree(current);
+
+    return visit;
+}
+
+VALUE
+pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze) {
+    VALUE constants = rb_ary_new_capa(parser->constant_pool.size);
+
+    for (uint32_t index = 0; index < parser->constant_pool.size; index++) {
+        pm_constant_t *constant = &parser->constant_pool.constants[index];
+        int state = 0;
+
+        VALUE string = rb_enc_str_new((const char *) constant->start, constant->length, encoding);
+        VALUE value = rb_protect(rb_str_intern, string, &state);
+
+        if (state != 0) {
+            value = ID2SYM(rb_intern_const("?"));
+            rb_set_errinfo(Qnil);
+        }
+
+        rb_ary_push(constants, value);
+    }
+
+    pm_node_stack_node_t *node_stack = NULL;
+    pm_node_stack_push(&node_stack, node);
+    VALUE value_stack = rb_ary_new();
+
+    while (node_stack != NULL) {
+        if (!node_stack->visited) {
+            if (node_stack->visit == NULL) {
+                pm_node_stack_pop(&node_stack);
+                rb_ary_push(value_stack, Qnil);
+                continue;
+            }
+
+            const pm_node_t *node = node_stack->visit;
+            node_stack->visited = true;
+
+            switch (PM_NODE_TYPE(node)) {
+                <%- nodes.each do |node| -%>
+                <%- if node.fields.any? { |field| [Prism::Template::NodeField, Prism::Template::OptionalNodeField, Prism::Template::NodeListField].include?(field.class) } -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                case <%= node.type %>: {
+                    pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
+                    <%- node.fields.each do |field| -%>
+                    <%- case field -%>
+                    <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%>
+                    pm_node_stack_push(&node_stack, (pm_node_t *) cast-><%= field.name %>);
+                    <%- when Prism::Template::NodeListField -%>
+                    for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
+                        pm_node_stack_push(&node_stack, (pm_node_t *) cast-><%= field.name %>.nodes[index]);
+                    }
+                    <%- end -%>
+                    <%- end -%>
+                    break;
+                }
+                <%- end -%>
+                <%- end -%>
+                default:
+                    break;
+            }
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+        } else {
+            const pm_node_t *node = pm_node_stack_pop(&node_stack);
+
+            switch (PM_NODE_TYPE(node)) {
+                <%- nodes.each do |node| -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                case <%= node.type %>: {
+                    <%- if node.fields.any? { |field| ![Prism::Template::NodeField, Prism::Template::OptionalNodeField].include?(field.class) } -%>
+                    pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
+                    <%- end -%>
+                    VALUE argv[<%= node.fields.length + 4 %>];
+
+                    // source
+                    argv[0] = source;
+
+                    // node_id
+                    argv[1] = ULONG2NUM(node->node_id);
+
+                    // location
+                    argv[2] = pm_location_new(parser, node->location.start, node->location.end, source, freeze);
+
+                    // flags
+                    argv[3] = ULONG2NUM(node->flags);
+                    <%- node.fields.each.with_index(4) do |field, index| -%>
+
+                    // <%= field.name %>
+                    <%- case field -%>
+                    <%- when Prism::Template::NodeField, Prism::Template::OptionalNodeField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = rb_ary_pop(value_stack);
+                    <%- when Prism::Template::NodeListField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size);
+                    for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
+                        rb_ary_push(argv[<%= index %>], rb_ary_pop(value_stack));
+                    }
+                    if (freeze) rb_obj_freeze(argv[<%= index %>]);
+                    <%- when Prism::Template::StringField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = pm_string_new(&cast-><%= field.name %>, encoding);
+                    <%- when Prism::Template::ConstantField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    assert(cast-><%= field.name %> != 0);
+                    argv[<%= index %>] = RARRAY_AREF(constants, cast-><%= field.name %> - 1);
+                    <%- when Prism::Template::OptionalConstantField -%>
+                    argv[<%= index %>] = cast-><%= field.name %> == 0 ? Qnil : RARRAY_AREF(constants, cast-><%= field.name %> - 1);
+                    <%- when Prism::Template::ConstantListField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size);
+                    for (size_t index = 0; index < cast-><%= field.name %>.size; index++) {
+                        assert(cast-><%= field.name %>.ids[index] != 0);
+                        rb_ary_push(argv[<%= index %>], RARRAY_AREF(constants, cast-><%= field.name %>.ids[index] - 1));
+                    }
+                    if (freeze) rb_obj_freeze(argv[<%= index %>]);
+                    <%- when Prism::Template::LocationField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
+                    <%- when Prism::Template::OptionalLocationField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze);
+                    <%- when Prism::Template::UInt8Field -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>);
+                    <%- when Prism::Template::UInt32Field -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = ULONG2NUM(cast-><%= field.name %>);
+                    <%- when Prism::Template::IntegerField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = pm_integer_new(&cast-><%= field.name %>);
+                    <%- when Prism::Template::DoubleField -%>
+#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
+                    argv[<%= index %>] = DBL2NUM(cast-><%= field.name %>);
+                    <%- else -%>
+                    <%- raise -%>
+                    <%- end -%>
+                    <%- end -%>
+
+                    VALUE value = rb_class_new_instance(<%= node.fields.length + 4 %>, argv, rb_cPrism<%= node.name %>);
+                    if (freeze) rb_obj_freeze(value);
+
+                    rb_ary_push(value_stack, value);
+                    break;
+                }
+                <%- end -%>
+                default:
+                    rb_raise(rb_eRuntimeError, "unknown node type: %d", PM_NODE_TYPE(node));
+            }
+        }
+    }
+
+    return rb_ary_pop(value_stack);
+}
+
+void
+Init_prism_api_node(void) {
+    <%- nodes.each do |node| -%>
+    rb_cPrism<%= node.name %> = rb_define_class_under(rb_cPrism, "<%= node.name %>", rb_cPrismNode);
+    <%- end -%>
+}