herb 0.11.0-x86-linux-gnu → 0.11.2-x86-linux-gnu

data/src/include/analyze/analyzed_ruby.h

@@ -3,11 +3,36 @@
 
 #include "../lib/hb_array.h"
 #include "../lib/hb_string.h"
+#include "../parser/parser_options.h"
 
 #include <prism.h>
+#include <stdbool.h>
+
+typedef enum {
+  CONTROL_TYPE_IF,
+  CONTROL_TYPE_ELSIF,
+  CONTROL_TYPE_ELSE,
+  CONTROL_TYPE_END,
+  CONTROL_TYPE_CASE,
+  CONTROL_TYPE_CASE_MATCH,
+  CONTROL_TYPE_WHEN,
+  CONTROL_TYPE_IN,
+  CONTROL_TYPE_BEGIN,
+  CONTROL_TYPE_RESCUE,
+  CONTROL_TYPE_ENSURE,
+  CONTROL_TYPE_UNLESS,
+  CONTROL_TYPE_WHILE,
+  CONTROL_TYPE_UNTIL,
+  CONTROL_TYPE_FOR,
+  CONTROL_TYPE_BLOCK,
+  CONTROL_TYPE_BLOCK_CLOSE,
+  CONTROL_TYPE_YIELD,
+  CONTROL_TYPE_UNKNOWN
+} control_type_t;
 
 typedef struct ANALYZED_RUBY_STRUCT {
-  pm_parser_t parser;
+  pm_arena_t* arena;
+  pm_parser_t* parser;
   pm_node_t* root;
   bool valid;
   bool parsed;
@@ -31,9 +56,11 @@ typedef struct ANALYZED_RUBY_STRUCT {
   int yield_node_count;
   int then_keyword_count;
   int unclosed_control_flow_count;
+  control_type_t control_type;
+  bool control_type_detected;
 } analyzed_ruby_T;
 
-analyzed_ruby_T* init_analyzed_ruby(hb_string_T source);
+analyzed_ruby_T* init_analyzed_ruby(hb_string_T source, const parser_options_T* options);
 void free_analyzed_ruby(analyzed_ruby_T* analyzed);
 hb_string_T erb_keyword_from_analyzed_ruby(const analyzed_ruby_T* analyzed);
 

data/src/include/analyze/helpers.h

@@ -37,34 +37,14 @@ bool control_role_split_offset(const analyzed_ruby_T* analyzed, uint32_t* offset
 
 bool has_error_message(analyzed_ruby_T* anlayzed, const char* message);
 
-bool is_do_block(pm_location_t opening_location);
-bool is_brace_block(pm_location_t opening_location);
-bool is_closing_brace(pm_location_t location);
-bool has_valid_block_closing(pm_location_t opening_loc, pm_location_t closing_loc);
+bool is_do_block(const uint8_t* source, pm_location_t opening_location);
+bool is_brace_block(const uint8_t* source, pm_location_t opening_location);
+bool is_closing_brace(const uint8_t* source, pm_location_t location);
+bool has_valid_block_closing(const uint8_t* source, pm_location_t opening_loc, pm_location_t closing_loc);
 
-bool search_begin_nodes(const pm_node_t* node, void* data);
-bool search_block_nodes(const pm_node_t* node, void* data);
-bool search_case_match_nodes(const pm_node_t* node, void* data);
-bool search_case_nodes(const pm_node_t* node, void* data);
-bool search_for_nodes(const pm_node_t* node, void* data);
-bool search_if_nodes(const pm_node_t* node, void* data);
-bool search_in_nodes(const pm_node_t* node, void* data);
-bool search_then_keywords(const pm_node_t* node, void* data);
-bool search_unclosed_control_flows(const pm_node_t* node, void* data);
-bool search_unless_nodes(const pm_node_t* node, void* data);
-bool search_until_nodes(const pm_node_t* node, void* data);
-bool search_when_nodes(const pm_node_t* node, void* data);
-bool search_while_nodes(const pm_node_t* node, void* data);
-bool search_yield_nodes(const pm_node_t* node, void* data);
-
-bool search_unexpected_block_closing_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_else_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_elsif_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_end_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_ensure_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_in_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_rescue_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_when_nodes(analyzed_ruby_T* analyzed);
+pm_visit_result_t search_control_flow_nodes(const pm_node_t* node, void* data);
+pm_visit_result_t search_unclosed_control_flows(const pm_node_t* node, void* data);
+void search_unexpected_keyword_errors(analyzed_ruby_T* analyzed);
 
 void check_erb_node_for_missing_end(const AST_NODE_T* node, hb_allocator_T* allocator, const parser_options_T* options);
 

data/src/include/ast/ast_nodes.h

@@ -478,6 +478,7 @@ typedef struct AST_ERB_YIELD_NODE_STRUCT {
   token_T* tag_opening;
   token_T* content;
   token_T* tag_closing;
+  herb_prism_node_T prism_node;
 } AST_ERB_YIELD_NODE_T;
 
 typedef struct AST_ERB_IN_NODE_STRUCT {
@@ -565,7 +566,7 @@ AST_RUBY_RENDER_KEYWORDS_NODE_T* ast_ruby_render_keywords_node_init(token_T* par
 AST_ERB_RENDER_NODE_T* ast_erb_render_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, analyzed_ruby_T* analyzed_ruby, herb_prism_node_T prism_node, struct AST_RUBY_RENDER_KEYWORDS_NODE_STRUCT* keywords, hb_array_T* body, hb_array_T* block_arguments, struct AST_ERB_RESCUE_NODE_STRUCT* rescue_clause, struct AST_ERB_ELSE_NODE_STRUCT* else_clause, struct AST_ERB_ENSURE_NODE_STRUCT* ensure_clause, struct AST_ERB_END_NODE_STRUCT* end_node, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_RUBY_PARAMETER_NODE_T* ast_ruby_parameter_node_init(token_T* name, struct AST_RUBY_LITERAL_NODE_STRUCT* default_value, hb_string_T kind, bool required, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_ERB_STRICT_LOCALS_NODE_T* ast_erb_strict_locals_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, analyzed_ruby_T* analyzed_ruby, herb_prism_node_T prism_node, hb_array_T* locals, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
-AST_ERB_YIELD_NODE_T* ast_erb_yield_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
+AST_ERB_YIELD_NODE_T* ast_erb_yield_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, herb_prism_node_T prism_node, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_ERB_IN_NODE_T* ast_erb_in_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, location_T* then_keyword, hb_array_T* statements, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_HERB_DIRECTIVE_NODE_T* ast_herb_directive_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, token_T* key, token_T* arguments, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_HERB_STATE_DECLARATION_NODE_T* ast_herb_state_declaration_node_init(token_T* name, struct AST_RUBY_LITERAL_NODE_STRUCT* default_value, hb_string_T kind, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);

data/src/include/herb.h

@@ -36,9 +36,10 @@ HERB_EXPORTED_FUNCTION const char* herb_version(void);
 HERB_EXPORTED_FUNCTION const char* herb_prism_version(void);
 
 typedef struct {
-  pm_parser_t parser;
+  pm_arena_t* arena;
+  pm_parser_t* parser;
   pm_node_t* root;
-  pm_options_t options;
+  pm_options_t* options;
 } herb_ruby_parse_result_T;
 
 HERB_EXPORTED_FUNCTION herb_ruby_parse_result_T* herb_parse_ruby(const char* source, size_t length);

data/src/include/lexer/lexer.h

@@ -7,5 +7,6 @@
 void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator);
 token_T* lexer_next_token(lexer_T* lexer);
 token_T* lexer_error(lexer_T* lexer, const char* message);
+void lexer_skip_data_to(lexer_T* lexer, uint32_t position);
 
 #endif

data/src/include/parser/parser_options.h

@@ -10,6 +10,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#define HERB_PARSER_OPTIONS_STRING_CAPACITY 16
+
 typedef struct PARSER_OPTIONS_STRUCT {
   bool action_view_helpers;
   bool analyze;
@@ -24,6 +26,7 @@ typedef struct PARSER_OPTIONS_STRUCT {
   bool prism_nodes_deep;
   bool prism_program;
   bool render_nodes;
+  char ruby_version[HERB_PARSER_OPTIONS_STRING_CAPACITY];
   bool strict;
   bool strict_locals;
   uint32_t timeout_ms;
@@ -38,4 +41,6 @@ typedef struct PARSER_OPTIONS_STRUCT {
 
 extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
 
+bool herb_ruby_version_supported(const char* value, size_t length);
+
 #endif

data/src/include/prism/herb_prism_node.h

@@ -2,12 +2,17 @@
 #define HERB_PRISM_NODE_H
 
 #include <prism.h>
+#include <stdint.h>
 
 typedef struct {
   pm_node_t* node;
   pm_parser_t* parser;
+  uint32_t source_start;
+  uint32_t source_end;
 } herb_prism_node_T;
 
-#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL })
+#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL, .source_start = 0, .source_end = 0 })
+
+void herb_prism_node_serialize(herb_prism_node_T prism_node, pm_buffer_t* buffer);
 
 #endif

data/src/include/prism/prism_context.h

@@ -7,14 +7,16 @@
 #include <stdbool.h>
 
 typedef struct {
-  pm_parser_t parser;
-  pm_options_t pm_opts;
+  pm_arena_t* arena;
+  pm_parser_t* parser;
+  pm_options_t* pm_opts;
   pm_node_t* root;
   hb_buffer_T ruby_buf;
 
   bool has_structural;
-  pm_parser_t structural_parser;
-  pm_options_t structural_pm_opts;
+  pm_arena_t* structural_arena;
+  pm_parser_t* structural_parser;
+  pm_options_t* structural_pm_opts;
   pm_node_t* structural_root;
   hb_buffer_T structural_buf;
 
@@ -24,17 +26,15 @@ typedef struct {
 static inline void herb_prism_context_free(herb_prism_context_T* context) {
   if (!context) { return; }
 
-  if (context->root) { pm_node_destroy(&context->parser, context->root); }
-
-  pm_parser_free(&context->parser);
-  pm_options_free(&context->pm_opts);
+  if (context->parser) { pm_parser_free(context->parser); }
+  if (context->arena) { pm_arena_free(context->arena); }
+  if (context->pm_opts) { pm_options_free(context->pm_opts); }
   hb_buffer_free(&context->ruby_buf);
 
   if (context->has_structural) {
-    if (context->structural_root) { pm_node_destroy(&context->structural_parser, context->structural_root); }
-
-    pm_parser_free(&context->structural_parser);
-    pm_options_free(&context->structural_pm_opts);
+    if (context->structural_parser) { pm_parser_free(context->structural_parser); }
+    if (context->structural_arena) { pm_arena_free(context->structural_arena); }
+    if (context->structural_pm_opts) { pm_options_free(context->structural_pm_opts); }
 
     if (context->structural_buf.value) { hb_buffer_free(&context->structural_buf); }
   }

data/src/include/prism/prism_helpers.h

@@ -7,9 +7,39 @@
 #include "../lib/hb_allocator.h"
 #include "../location/location.h"
 #include "../location/position.h"
+#include "../parser/parser_options.h"
 
 #include <prism.h>
 
+static inline bool herb_prism_location_present(pm_location_t location) {
+  return location.length > 0;
+}
+
+static inline bool herb_prism_location_assigned(pm_location_t location) {
+  return location.start != 0 || location.length != 0;
+}
+
+static inline uint32_t herb_prism_location_end(pm_location_t location) {
+  return location.start + location.length;
+}
+
+static inline const uint8_t* herb_prism_location_start_pointer(const uint8_t* base, pm_location_t location) {
+  return base + location.start;
+}
+
+static inline const uint8_t* herb_prism_location_end_pointer(const uint8_t* base, pm_location_t location) {
+  return base + location.start + location.length;
+}
+
+void herb_prism_options_set_ruby_version(pm_options_t* options, const parser_options_T* parser_options);
+
+pm_parser_t* herb_prism_parser_new(
+  pm_arena_t* arena,
+  const uint8_t* source,
+  size_t length,
+  const parser_options_T* parser_options
+);
+
 hb_string_T pm_error_level_to_string(pm_error_level_t level);
 
 RUBY_PARSE_ERROR_T* ruby_parse_error_from_prism_error(

data/src/include/prism_version.h

@@ -0,0 +1,11 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See templates/src/include/prism_version.h.erb
+
+#ifndef HERB_PRISM_VERSION_H
+#define HERB_PRISM_VERSION_H
+
+#include <prism/version.h>
+
+#define HERB_PRISM_VERSION PRISM_VERSION "+4691667"
+
+#endif

data/src/include/version.h

@@ -1,6 +1,6 @@
 #ifndef HERB_VERSION_H
 #define HERB_VERSION_H
 
-#define HERB_VERSION "0.11.0"
+#define HERB_VERSION "0.11.2"
 
 #endif

data/src/lexer.c

@@ -430,6 +430,38 @@ static token_T* lexer_parse_erb_close(lexer_T* lexer) {
   return lexer_advance_with(lexer, hb_string("%>"), TOKEN_ERB_END);
 }
 
+void lexer_skip_data_to(lexer_T* lexer, uint32_t position) {
+  if (position > lexer->source.length) { position = lexer->source.length; }
+
+  while (lexer->current_position < position && !lexer_eof(lexer)) {
+    char character = lexer->current_character;
+
+    if (character == '\r' && lexer_peek(lexer, 1) == '\n') {
+      lexer->current_position += 2;
+      lexer->current_line++;
+      lexer->current_column = 0;
+    } else if (is_newline(character)) {
+      lexer->current_position++;
+      lexer->current_line++;
+      lexer->current_column = 0;
+    } else if ((unsigned char) character < 0x80) {
+      lexer->current_position++;
+      lexer->current_column++;
+    } else {
+      uint32_t length = utf8_sequence_length(hb_string_slice(lexer->source, lexer->current_position));
+
+      lexer->current_position += length > 0 ? length : 1;
+      lexer->current_column++;
+    }
+
+    lexer->current_character = lexer->source.data[lexer->current_position];
+  }
+
+  lexer->previous_line = lexer->current_line;
+  lexer->previous_column = lexer->current_column;
+  lexer->previous_position = lexer->current_position;
+}
+
 // ===== Tokenizing Function
 
 token_T* lexer_next_token(lexer_T* lexer) {

data/src/parser/parser_helpers.c

@@ -195,8 +195,7 @@ void parser_append_literal_node_from_buffer(
 
   if (children != NULL) { hb_array_append(children, literal); }
 
-  hb_buffer_free(buffer);
-  hb_buffer_init(buffer, 128, parser->allocator);
+  hb_buffer_clear(buffer);
 }
 
 token_T* parser_advance(parser_T* parser) {

data/src/parser.c

@@ -139,7 +139,7 @@ static AST_HTML_COMMENT_NODE_T* parser_parse_html_comment(parser_T* parser) {
   position_T start = parser->current_token->location.start;
 
   hb_buffer_T comment;
-  hb_buffer_init(&comment, 512, parser->allocator);
+  hb_buffer_init(&comment, 64, parser->allocator);
 
   token_T* comment_end = NULL;
 
@@ -376,7 +376,7 @@ static AST_HTML_TEXT_NODE_T* parser_parse_text_content(parser_T* parser, hb_arra
   position_T start = parser->current_token->location.start;
 
   hb_buffer_T content;
-  hb_buffer_init(&content, 2048, parser->allocator);
+  hb_buffer_init(&content, 64, parser->allocator);
 
   while (token_is_none_of(
     parser,
@@ -553,7 +553,7 @@ static AST_HTML_ATTRIBUTE_VALUE_NODE_T* parser_parse_quoted_html_attribute_value
   hb_array_T** errors
 ) {
   hb_buffer_T buffer;
-  hb_buffer_init(&buffer, 512, parser->allocator);
+  hb_buffer_init(&buffer, 64, parser->allocator);
   token_T* opening_quote = parser_consume_expected(parser, TOKEN_QUOTE, errors);
   position_T start = parser->current_token->location.start;
 
@@ -816,7 +816,7 @@ static AST_HTML_ATTRIBUTE_NODE_T* parser_parse_html_attribute(parser_T* parser)
 
     if (has_equals) {
       hb_buffer_T equals_buffer;
-      hb_buffer_init(&equals_buffer, 256, parser->allocator);
+      hb_buffer_init(&equals_buffer, 16, parser->allocator);
       position_T equals_start = { 0 };
       position_T equals_end = { 0 };
       uint32_t range_start = 0;
@@ -1640,13 +1640,12 @@ static void parser_append_foreign_content_from_buffer(
     ast_html_text_node_init(content, start, parser->current_token->location.start, NULL, parser->allocator)
   );
 
-  hb_buffer_free(buffer);
-  hb_buffer_init(buffer, 128, parser->allocator);
+  hb_buffer_clear(buffer);
 }
 
 static void parser_parse_foreign_content(parser_T* parser, hb_array_T* children, hb_array_T** errors) {
   hb_buffer_T content;
-  hb_buffer_init(&content, 1024, parser->allocator);
+  hb_buffer_init(&content, 64, parser->allocator);
   position_T start = parser->current_token->location.start;
   bool has_closing_tag = !hb_string_is_empty(parser->foreign_content_tag_name);
 

data/src/parser_options_defaults.c

@@ -18,6 +18,7 @@ const parser_options_T HERB_DEFAULT_PARSER_OPTIONS = { .action_view_helpers = fa
                                                        .prism_nodes_deep = false,
                                                        .prism_program = false,
                                                        .render_nodes = false,
+                                                       .ruby_version = "4.0",
                                                        .strict = true,
                                                        .strict_locals = false,
                                                        .timeout_ms = 1000,