herb 0.10.3 → 0.11.0

data/src/lexer.c

@@ -2,6 +2,7 @@
 #include "include/lexer/token.h"
 #include "include/lib/hb_string.h"
 #include "include/macros.h"
+#include "include/prism/ruby_parser.h"
 #include "include/util/utf8.h"
 #include "include/util/util.h"
 
@@ -10,8 +11,26 @@
 #include <string.h>
 
 #define LEXER_STALL_LIMIT 5
+#define LEXER_ERB_END_CANDIDATE_LIMIT 16
+#define LEXER_ERB_SPACED_DELIMITER_LIMIT 8
 
-static hb_string_T erb_open_patterns[] = HB_STRING_LIST("<%==", "<%%=", "<%graphql", "<%=", "<%#", "<%-", "<%%", "<%");
+typedef enum {
+  ERB_END_CANDIDATE_PERCENT,
+  ERB_END_CANDIDATE_HTML_TAG,
+  ERB_END_CANDIDATE_ANGLE_BRACKET,
+} erb_end_candidate_kind_T;
+
+typedef struct {
+  uint32_t position;
+  uint32_t line;
+  uint32_t column;
+  uint8_t delimiter_length;
+  erb_end_candidate_kind_T kind;
+} erb_end_candidate_T;
+
+const hb_string_T HERB_DEFAULT_ERB_OPENINGS[] = HB_STRING_LIST("<%", "<%=", "<%==", "<%#", "<%-", "<%%", "<%%=");
+
+const size_t HERB_DEFAULT_ERB_OPENINGS_COUNT = sizeof(HERB_DEFAULT_ERB_OPENINGS) / sizeof(HERB_DEFAULT_ERB_OPENINGS[0]);
 
 static bool lexer_eof(const lexer_T* lexer) {
   return lexer->current_character == '\0' || lexer->stalled;
@@ -45,6 +64,7 @@ void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator) {
 
   lexer->current_character = lexer->source.data[0];
   lexer->state = STATE_DATA;
+  lexer->erb_openers = (erb_openers_T) { .items = NULL, .count = 0 };
 
   lexer->current_line = 1;
   lexer->current_column = 0;
@@ -57,6 +77,8 @@ void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator) {
   lexer->stall_counter = 0;
   lexer->last_position = 0;
   lexer->stalled = false;
+  lexer->in_erb_comment = false;
+  lexer->malformed_erb_close_length = 0;
 }
 
 token_T* lexer_error(lexer_T* lexer, const char* message) {
@@ -191,29 +213,145 @@ static token_T* lexer_parse_identifier(lexer_T* lexer) {
 
 // ===== ERB Parsing
 
+static bool is_erb_opener_word_character(const char character) {
+  return isalnum(character) || character == '_';
+}
+
+static bool lexer_matches_erb_opener(const lexer_T* lexer, hb_string_T opener) {
+  if (hb_string_is_empty(opener)) { return false; }
+
+  hb_string_T remaining_source = hb_string_slice(lexer->source, lexer->current_position + 2);
+
+  if (!hb_string_starts_with(remaining_source, opener)) { return false; }
+  if (!is_erb_opener_word_character(opener.data[opener.length - 1])) { return true; }
+
+  return !is_erb_opener_word_character(lexer_peek(lexer, 2 + opener.length));
+}
+
 static token_T* lexer_parse_erb_open(lexer_T* lexer) {
   lexer->state = STATE_ERB_CONTENT;
-  for (size_t i = 0; i < sizeof(erb_open_patterns) / sizeof(erb_open_patterns[0]); i++) {
-    token_T* match = lexer_match_and_advance(lexer, erb_open_patterns[i], TOKEN_ERB_START);
-    if (match) { return match; }
+  lexer->in_erb_comment = false;
+
+  if (lexer_peek(lexer, 2) == '%' && lexer_peek(lexer, 3) == '>') {
+    return lexer_advance_with(lexer, hb_string("<%"), TOKEN_ERB_START);
+  }
+
+  hb_string_T remaining_source = hb_string_slice(lexer->source, lexer->current_position);
+  hb_string_T longest_default = HB_STRING_NULL;
+  uint32_t custom_length = 0;
+
+  for (size_t i = 0; i < HERB_DEFAULT_ERB_OPENINGS_COUNT; i++) {
+    hb_string_T opening = HERB_DEFAULT_ERB_OPENINGS[i];
+
+    if (hb_string_starts_with(remaining_source, opening) && opening.length > longest_default.length) {
+      longest_default = opening;
+    }
+  }
+
+  for (size_t i = 0; i < lexer->erb_openers.count; i++) {
+    hb_string_T opener = lexer->erb_openers.items[i];
+
+    if (lexer_matches_erb_opener(lexer, opener) && opener.length + 2 > custom_length) {
+      custom_length = opener.length + 2;
+    }
+  }
+
+  if (custom_length > longest_default.length) { return lexer_advance_with_next(lexer, custom_length, TOKEN_ERB_START); }
+
+  if (longest_default.length > 0) {
+    lexer->in_erb_comment = hb_string_equals(longest_default, hb_string("<%#"));
+
+    return lexer_advance_with(lexer, longest_default, TOKEN_ERB_START);
   }
 
   return lexer_error(lexer, "Unexpected ERB start");
 }
 
+static uint8_t lexer_erb_percent_delimiter_length(const lexer_T* lexer) {
+  if (lexer->current_character != '%') { return 0; }
+
+  char next = lexer_peek(lexer, 1);
+
+  if (is_newline(next) || next == '<' || next == '"' || next == '\'' || next == '\0') { return 1; }
+
+  for (uint8_t offset = 1; offset <= LEXER_ERB_SPACED_DELIMITER_LIMIT; offset++) {
+    char character = lexer_peek(lexer, offset);
+
+    if (character == '>') { return offset + 1; }
+    if (character != ' ' && character != '\t') { break; }
+  }
+
+  return 0;
+}
+
+static bool lexer_recover_erb_tag_end(
+  lexer_T* lexer,
+  uint32_t start_position,
+  const erb_end_candidate_T* candidates,
+  size_t candidate_count
+) {
+  for (int pass = ERB_END_CANDIDATE_PERCENT; pass <= ERB_END_CANDIDATE_ANGLE_BRACKET; pass++) {
+    bool last_first = (pass == ERB_END_CANDIDATE_ANGLE_BRACKET);
+
+    for (size_t offset = 0; offset < candidate_count; offset++) {
+      const erb_end_candidate_T* candidate = &candidates[last_first ? candidate_count - 1 - offset : offset];
+
+      if (candidate->kind != (erb_end_candidate_kind_T) pass) { continue; }
+
+      if (!herb_ruby_fragment_is_parseable(hb_string_range(lexer->source, start_position, candidate->position))) {
+        continue;
+      }
+
+      lexer->current_position = candidate->position;
+      lexer->current_line = candidate->line;
+      lexer->current_column = candidate->column;
+      lexer->current_character = lexer->source.data[candidate->position];
+
+      if (candidate->delimiter_length > 0) {
+        lexer->state = STATE_ERB_CLOSE;
+        lexer->malformed_erb_close_length = candidate->delimiter_length;
+      } else {
+        lexer->state = STATE_DATA;
+      }
+
+      return true;
+    }
+  }
+
+  return false;
+}
+
+static bool lexer_erb_content_spans_nested_start(lexer_T* lexer, uint32_t start_position) {
+  for (uint32_t position = lexer->current_position; position + 1 < lexer->source.length; position++) {
+    if (lexer->source.data[position] != '%' || lexer->source.data[position + 1] != '>') { continue; }
+
+    return herb_ruby_fragment_is_parseable(hb_string_range(lexer->source, start_position, position));
+  }
+
+  return false;
+}
+
 static token_T* lexer_parse_erb_content(lexer_T* lexer) {
   uint32_t start_position = lexer->current_position;
 
+  erb_end_candidate_T candidates[LEXER_ERB_END_CANDIDATE_LIMIT];
+  size_t candidate_count = 0;
+
   while (!lexer_peek_erb_end(lexer, 0)) {
-    if (lexer_eof(lexer)) {
-      token_T* token =
-        token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_ERB_CONTENT, lexer);
+    bool nested_start = !lexer->in_erb_comment && lexer_peek_erb_start(lexer, 0);
 
-      return token;
-    }
+    if (lexer_eof(lexer) || nested_start) {
+      if (!lexer_recover_erb_tag_end(lexer, start_position, candidates, candidate_count)) {
+        if (nested_start && lexer_erb_content_spans_nested_start(lexer, start_position)) {
+          lexer->current_position += 2;
+          lexer->current_column += 2;
+          lexer->current_character = lexer->source.data[lexer->current_position];
+
+          continue;
+        }
 
-    if (lexer_peek_erb_start(lexer, 0)) {
-      lexer->state = STATE_DATA;
+        if (!lexer_eof(lexer)) { lexer->state = STATE_DATA; }
+      }
 
       token_T* token =
         token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_ERB_CONTENT, lexer);
@@ -221,10 +359,44 @@ static token_T* lexer_parse_erb_content(lexer_T* lexer) {
       return token;
     }
 
+    if (candidate_count < LEXER_ERB_END_CANDIDATE_LIMIT) {
+      char next = lexer_peek(lexer, 1);
+      uint8_t trailing = lexer_erb_percent_delimiter_length(lexer);
+
+      if (trailing > 0) {
+        uint8_t leading = 0;
+
+        if (lexer->current_position > start_position) {
+          char preceding = lexer->source.data[lexer->current_position - 1];
+
+          if (preceding == '-' || preceding == '=' || preceding == '%') { leading = 1; }
+        }
+
+        candidates[candidate_count++] = (erb_end_candidate_T) { .position = lexer->current_position - leading,
+                                                                .line = lexer->current_line,
+                                                                .column = lexer->current_column - leading,
+                                                                .delimiter_length = leading + trailing,
+                                                                .kind = ERB_END_CANDIDATE_PERCENT };
+      } else if (lexer->current_character == '<'
+                 && (isalpha(next) || next == '!' || (next == '/' && isalpha(lexer_peek(lexer, 2))))) {
+        candidates[candidate_count++] = (erb_end_candidate_T) { .position = lexer->current_position,
+                                                                .line = lexer->current_line,
+                                                                .column = lexer->current_column,
+                                                                .delimiter_length = 0,
+                                                                .kind = ERB_END_CANDIDATE_HTML_TAG };
+      } else if (lexer->current_character == '>') {
+        candidates[candidate_count++] = (erb_end_candidate_T) { .position = lexer->current_position,
+                                                                .line = lexer->current_line,
+                                                                .column = lexer->current_column,
+                                                                .delimiter_length = 1,
+                                                                .kind = ERB_END_CANDIDATE_ANGLE_BRACKET };
+      }
+    }
+
     if (is_newline(lexer->current_character)) {
       lexer->current_line++;
       lexer->current_column = 0;
-    } else {
+    } else if (!utf8_is_valid_continuation_byte((unsigned char) lexer->current_character)) {
       lexer->current_column++;
     }
 
@@ -242,6 +414,14 @@ static token_T* lexer_parse_erb_content(lexer_T* lexer) {
 
 static token_T* lexer_parse_erb_close(lexer_T* lexer) {
   lexer->state = STATE_DATA;
+  lexer->in_erb_comment = false;
+
+  if (lexer->malformed_erb_close_length > 0) {
+    uint8_t length = lexer->malformed_erb_close_length;
+    lexer->malformed_erb_close_length = 0;
+
+    return lexer_advance_with_next(lexer, length, TOKEN_ERB_END);
+  }
 
   if (lexer_peek_erb_percent_close_tag(lexer, 0)) { return lexer_advance_with(lexer, hb_string("%%>"), TOKEN_ERB_END); }
   if (lexer_peek_erb_equals_close_tag(lexer, 0)) { return lexer_advance_with(lexer, hb_string("=%>"), TOKEN_ERB_END); }
@@ -283,6 +463,10 @@ token_T* lexer_next_token(lexer_T* lexer) {
         return lexer_advance_with_next(lexer, strlen("<?xml"), TOKEN_XML_DECLARATION);
       }
 
+      if (lexer_peek_for_xml_processing_instruction(lexer, 0)) {
+        return lexer_advance_with(lexer, hb_string("<?"), TOKEN_XML_PROCESSING_INSTRUCTION_START);
+      }
+
       if (lexer_peek_for_cdata_start(lexer, 0)) {
         return lexer_advance_with_next(lexer, strlen("<![CDATA["), TOKEN_CDATA_START);
       }

data/src/lib/hb_allocator.c

@@ -238,6 +238,8 @@ static void* tracking_realloc(hb_allocator_T* self, void* pointer, size_t _old_s
 }
 
 static void tracking_dealloc(hb_allocator_T* self, void* pointer) {
+  if (!pointer) { return; }
+
   hb_allocator_tracking_stats_T* stats = (hb_allocator_tracking_stats_T*) self->context;
   tracking_unrecord(stats, pointer);
   free(pointer);

data/src/lib/hb_array.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <stdint.h>
 #include <stdio.h>
 
@@ -33,6 +34,8 @@ hb_array_T* hb_array_init(const size_t capacity, hb_allocator_T* allocator) {
 }
 
 bool hb_array_append(hb_array_T* array, void* item) {
+  assert(array != NULL);
+
   if (!array) { return false; }
 
   if (array->size >= array->capacity) {
@@ -116,6 +119,15 @@ void hb_array_remove_item(hb_array_T* array, void* item) {
   if (index != SIZE_MAX) { hb_array_remove(array, index); }
 }
 
+bool hb_array_append_lazy(hb_array_T** array, void* item, hb_allocator_T* allocator) {
+  if (*array == NULL) {
+    *array = hb_array_init(8, allocator);
+    if (!*array) { return false; }
+  }
+
+  return hb_array_append(*array, item);
+}
+
 // Alias for hb_array_append
 bool hb_array_push(hb_array_T* array, void* item) {
   return hb_array_append(array, item);
@@ -141,6 +153,20 @@ size_t hb_array_capacity(const hb_array_T* array) {
   return array->capacity;
 }
 
+void hb_array_replace_contents(hb_array_T* array, hb_array_T** source) {
+  if (!array || !source || !*source) { return; }
+
+  hb_allocator_dealloc(array->allocator, array->items);
+
+  array->items = (*source)->items;
+  array->size = (*source)->size;
+  array->capacity = (*source)->capacity;
+
+  hb_allocator_dealloc((*source)->allocator, *source);
+
+  *source = NULL;
+}
+
 void hb_array_free(hb_array_T** array) {
   if (!array || !*array) { return; }
 

data/src/lib/hb_string.c

@@ -63,6 +63,32 @@ bool hb_string_is_blank(hb_string_T string) {
   return true;
 }
 
+bool hb_string_equals_collapsing_whitespace(hb_string_T a, hb_string_T b) {
+  uint32_t a_offset = 0;
+  uint32_t b_offset = 0;
+
+  while (a_offset < a.length && b_offset < b.length) {
+    if (is_whitespace(a.data[a_offset]) && is_whitespace(b.data[b_offset])) {
+      while (a_offset < a.length && is_whitespace(a.data[a_offset])) {
+        a_offset++;
+      }
+
+      while (b_offset < b.length && is_whitespace(b.data[b_offset])) {
+        b_offset++;
+      }
+
+      continue;
+    }
+
+    if (a.data[a_offset] != b.data[b_offset]) { return false; }
+
+    a_offset++;
+    b_offset++;
+  }
+
+  return a_offset == a.length && b_offset == b.length;
+}
+
 hb_string_T hb_string_copy(hb_string_T string, hb_allocator_T* allocator) {
   if (hb_string_is_null(string)) { return HB_STRING_NULL; }
   if (hb_string_is_empty(string)) { return HB_STRING_EMPTY; }

data/src/main.c

@@ -46,7 +46,7 @@ int main(const int argc, char* argv[]) {
   if (argc < 2) {
     puts("./herb [command] [options]\n");
 
-    puts("Herb 🌿 Powerful and seamless HTML-aware ERB toolchain.\n");
+    puts("Herb 🌿 A modern templating language for the HTML+ERB you already have.\n");
 
     puts("./herb lex [file]      -  Lex a file");
     puts("./herb parse [file]    -  Parse a file");

data/src/parser/dot_notation.c

@@ -14,7 +14,7 @@ static bool current_token_is_dot(parser_T* parser) {
   return token_is_dot(parser->current_token);
 }
 
-static void consume_dot_notation_loop(parser_T* parser, token_T* tag_name, hb_array_T* errors) {
+static void consume_dot_notation_loop(parser_T* parser, token_T* tag_name, hb_array_T** errors) {
   while (current_token_is_dot(parser)) {
     token_T* dot = parser_advance(parser);
     token_T* segment = parser_consume_expected(parser, TOKEN_IDENTIFIER, errors);
@@ -81,7 +81,7 @@ bool parser_lookahead_is_valid_dot_notation_open_tag(parser_T* parser) {
   return true;
 }
 
-void parser_consume_dot_notation_segments(parser_T* parser, token_T* tag_name, hb_array_T* errors) {
+void parser_consume_dot_notation_segments(parser_T* parser, token_T* tag_name, hb_array_T** errors) {
   if (!parser->options.dot_notation_tags) { return; }
   if (tag_name == NULL || tag_name->value.length == 0) { return; }
   if (!current_token_is_dot(parser)) { return; }
@@ -92,7 +92,8 @@ void parser_consume_dot_notation_segments(parser_T* parser, token_T* tag_name, h
       tag_name->location.start,
       tag_name->location.end,
       parser->allocator,
-      errors
+      errors,
+      &parser->options
     );
   }
 

data/src/parser/match_tags.c

@@ -1,5 +1,5 @@
 // NOTE: This file is generated by the templates/template.rb script and should not
-// be modified manually. See /home/runner/work/herb/herb/templates/src/parser/match_tags.c.erb
+// be modified manually. See templates/src/parser/match_tags.c.erb
 
 #include "../include/parser/parser.h"
 #include "../include/ast/ast_nodes.h"
@@ -157,6 +157,16 @@ bool match_tags_visitor(const AST_NODE_T* node, void* data) {
 
 
 
+    case AST_XML_PROCESSING_INSTRUCTION_NODE: {
+      const AST_XML_PROCESSING_INSTRUCTION_NODE_T* xml_processing_instruction_node = (const AST_XML_PROCESSING_INSTRUCTION_NODE_T*) node;
+
+      if (xml_processing_instruction_node->children != NULL) {
+        match_tags_in_node_array(xml_processing_instruction_node->children, context->errors, context->options, context->allocator);
+      }
+    } break;
+
+
+
     case AST_CDATA_NODE: {
       const AST_CDATA_NODE_T* cdata_node = (const AST_CDATA_NODE_T*) node;
 
@@ -224,6 +234,34 @@ bool match_tags_visitor(const AST_NODE_T* node, void* data) {
 
 
 
+    case AST_ERB_ITERATION_BLOCK_NODE: {
+      const AST_ERB_ITERATION_BLOCK_NODE_T* erb_iteration_block_node = (const AST_ERB_ITERATION_BLOCK_NODE_T*) node;
+
+      if (erb_iteration_block_node->arguments != NULL) {
+        match_tags_in_node_array(erb_iteration_block_node->arguments, context->errors, context->options, context->allocator);
+      }
+      if (erb_iteration_block_node->body != NULL) {
+        match_tags_in_node_array(erb_iteration_block_node->body, context->errors, context->options, context->allocator);
+      }
+      if (erb_iteration_block_node->block_arguments != NULL) {
+        match_tags_in_node_array(erb_iteration_block_node->block_arguments, context->errors, context->options, context->allocator);
+      }
+      if (erb_iteration_block_node->rescue_clause != NULL) {
+        herb_visit_node((AST_NODE_T*) erb_iteration_block_node->rescue_clause, match_tags_visitor, context);
+      }
+      if (erb_iteration_block_node->else_clause != NULL) {
+        herb_visit_node((AST_NODE_T*) erb_iteration_block_node->else_clause, match_tags_visitor, context);
+      }
+      if (erb_iteration_block_node->ensure_clause != NULL) {
+        herb_visit_node((AST_NODE_T*) erb_iteration_block_node->ensure_clause, match_tags_visitor, context);
+      }
+      if (erb_iteration_block_node->end_node != NULL) {
+        herb_visit_node((AST_NODE_T*) erb_iteration_block_node->end_node, match_tags_visitor, context);
+      }
+    } break;
+
+
+
     case AST_ERB_WHEN_NODE: {
       const AST_ERB_WHEN_NODE_T* erb_when_node = (const AST_ERB_WHEN_NODE_T*) node;
 
@@ -388,6 +426,9 @@ bool match_tags_visitor(const AST_NODE_T* node, void* data) {
       if (ruby_render_keywords_node->locals != NULL) {
         match_tags_in_node_array(ruby_render_keywords_node->locals, context->errors, context->options, context->allocator);
       }
+      if (ruby_render_keywords_node->state != NULL) {
+        match_tags_in_node_array(ruby_render_keywords_node->state, context->errors, context->options, context->allocator);
+      }
     } break;
 
 
@@ -450,6 +491,30 @@ bool match_tags_visitor(const AST_NODE_T* node, void* data) {
       }
     } break;
 
+
+
+
+
+    case AST_HERB_STATE_DECLARATION_NODE: {
+      const AST_HERB_STATE_DECLARATION_NODE_T* herb_state_declaration_node = (const AST_HERB_STATE_DECLARATION_NODE_T*) node;
+
+      if (herb_state_declaration_node->default_value != NULL) {
+        herb_visit_node((AST_NODE_T*) herb_state_declaration_node->default_value, match_tags_visitor, context);
+      }
+    } break;
+
+
+
+    case AST_HERB_STATE_DIRECTIVE_NODE: {
+      const AST_HERB_STATE_DIRECTIVE_NODE_T* herb_state_directive_node = (const AST_HERB_STATE_DIRECTIVE_NODE_T*) node;
+
+      if (herb_state_directive_node->states != NULL) {
+        match_tags_in_node_array(herb_state_directive_node->states, context->errors, context->options, context->allocator);
+      }
+    } break;
+
+
+
     default: break;
   }