herb 0.8.10-arm-linux-gnu → 0.9.0-arm-linux-gnu

data/src/include/extract.h

@@ -1,19 +1,35 @@
 #ifndef HERB_EXTRACT_H
 #define HERB_EXTRACT_H
 
+#include "util/hb_allocator.h"
 #include "util/hb_buffer.h"
 
+#include <stdbool.h>
+
 typedef enum {
   HERB_EXTRACT_LANGUAGE_RUBY,
   HERB_EXTRACT_LANGUAGE_HTML,
 } herb_extract_language_T;
 
-void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output);
-void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output);
+typedef struct {
+  bool semicolons;
+  bool comments;
+  bool preserve_positions;
+} herb_extract_ruby_options_T;
+
+extern const herb_extract_ruby_options_T HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+
+void herb_extract_ruby_to_buffer_with_options(
+  const char* source,
+  hb_buffer_T* output,
+  const herb_extract_ruby_options_T* options,
+  hb_allocator_T* allocator
+);
+void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator);
+void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator);
 
-char* herb_extract_ruby_with_semicolons(const char* source);
+char* herb_extract_ruby_with_semicolons(const char* source, hb_allocator_T* allocator);
 
-char* herb_extract(const char* source, herb_extract_language_T language);
-char* herb_extract_from_file(const char* path, herb_extract_language_T language);
+char* herb_extract(const char* source, herb_extract_language_T language, hb_allocator_T* allocator);
 
 #endif

data/src/include/herb.h

@@ -5,26 +5,38 @@
 #include "extract.h"
 #include "macros.h"
 #include "parser.h"
+#include "util/hb_allocator.h"
 #include "util/hb_array.h"
 #include "util/hb_buffer.h"
 
+#include <prism.h>
+#include <stdbool.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-HERB_EXPORTED_FUNCTION void herb_lex_to_buffer(const char* source, hb_buffer_T* output);
+HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source, hb_allocator_T* allocator);
 
-HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source);
-HERB_EXPORTED_FUNCTION hb_array_T* herb_lex_file(const char* path);
-
-HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(const char* source, parser_options_T* options);
+HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(
+  const char* source,
+  const parser_options_T* options,
+  hb_allocator_T* allocator
+);
 
 HERB_EXPORTED_FUNCTION const char* herb_version(void);
 HERB_EXPORTED_FUNCTION const char* herb_prism_version(void);
 
-HERB_EXPORTED_FUNCTION void herb_free_tokens(hb_array_T** tokens);
+typedef struct {
+  pm_parser_t parser;
+  pm_node_t* root;
+  pm_options_t options;
+} herb_ruby_parse_result_T;
+
+HERB_EXPORTED_FUNCTION herb_ruby_parse_result_T* herb_parse_ruby(const char* source, size_t length);
+HERB_EXPORTED_FUNCTION void herb_free_ruby_parse_result(herb_ruby_parse_result_T* result);
+HERB_EXPORTED_FUNCTION void herb_free_tokens(hb_array_T** tokens, hb_allocator_T* allocator);
 
 #ifdef __cplusplus
 }

data/src/include/herb_prism_node.h

@@ -0,0 +1,13 @@
+#ifndef HERB_PRISM_NODE_H
+#define HERB_PRISM_NODE_H
+
+#include <prism.h>
+
+typedef struct {
+  pm_node_t* node;
+  pm_parser_t* parser;
+} herb_prism_node_T;
+
+#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL })
+
+#endif

data/src/include/html_util.h

@@ -4,9 +4,14 @@
 #include "util/hb_string.h"
 #include <stdbool.h>
 
+struct hb_allocator;
+
 bool is_void_element(hb_string_T tag_name);
+bool has_optional_end_tag(hb_string_T tag_name);
+bool should_implicitly_close(hb_string_T open_tag_name, hb_string_T next_tag_name);
+bool parent_closes_element(hb_string_T open_tag_name, hb_string_T parent_close_tag_name);
 
-hb_string_T html_closing_tag_string(hb_string_T tag_name);
-hb_string_T html_self_closing_tag_string(hb_string_T tag_name);
+hb_string_T html_closing_tag_string(hb_string_T tag_name, struct hb_allocator* allocator);
+hb_string_T html_self_closing_tag_string(hb_string_T tag_name, struct hb_allocator* allocator);
 
 #endif

data/src/include/io.h

@@ -4,6 +4,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-char* herb_read_file(const char* filename);
+struct hb_allocator;
+
+char* herb_read_file(const char* filename, struct hb_allocator* allocator);
 
 #endif

data/src/include/lex_helpers.h

@@ -0,0 +1,29 @@
+#ifndef HERB_LEX_HELPERS_H
+#define HERB_LEX_HELPERS_H
+
+#include "herb.h"
+#include "token.h"
+#include "util/hb_allocator.h"
+#include "util/hb_array.h"
+#include "util/hb_buffer.h"
+#include "util/hb_string.h"
+
+#include <stdlib.h>
+
+static inline void herb_lex_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {
+  hb_array_T* tokens = herb_lex(source, allocator);
+
+  for (size_t i = 0; i < hb_array_size(tokens); i++) {
+    token_T* token = hb_array_get(tokens, i);
+
+    hb_string_T type = token_to_string(allocator, token);
+    hb_buffer_append_string(output, type);
+    hb_allocator_dealloc(allocator, type.data);
+
+    hb_buffer_append(output, "\n");
+  }
+
+  herb_free_tokens(&tokens, allocator);
+}
+
+#endif

data/src/include/lexer.h

@@ -4,7 +4,7 @@
 #include "lexer_struct.h"
 #include "token_struct.h"
 
-void lexer_init(lexer_T* lexer, const char* source);
+void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator);
 token_T* lexer_next_token(lexer_T* lexer);
 token_T* lexer_error(lexer_T* lexer, const char* message);
 

data/src/include/lexer_peek_helpers.h

@@ -2,8 +2,11 @@
 #define HERB_LEXER_PEEK_HELPERS_H
 
 #include "lexer_struct.h"
+#include "macros.h"
 #include "token_struct.h"
+#include "util/hb_string.h"
 
+#include <ctype.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -20,27 +23,98 @@ typedef struct {
   lexer_state_T state;
 } lexer_state_snapshot_T;
 
-char lexer_peek(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_cdata_end(const lexer_T* lexer, uint32_t offset);
-
 bool lexer_peek_for_html_comment_start(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type);
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset);
 
-bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset);
+static inline char lexer_peek(const lexer_T* lexer, uint32_t offset) {
+  return lexer->source.data[MIN(lexer->current_position + offset, lexer->source.length)];
+}
 
-char lexer_backtrack(const lexer_T* lexer, uint32_t offset);
+static inline char lexer_backtrack(const lexer_T* lexer, uint32_t offset) {
+  return lexer->source.data[MAX(lexer->current_position - offset, 0)];
+}
 
-bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type);
-bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset);
+static inline bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '-'
+      && lexer->source.data[position + 1] == '-' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_for_html_comment_invalid_end(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 3 < lexer->source.length && lexer->source.data[position] == '-'
+      && lexer->source.data[position + 1] == '-' && lexer->source.data[position + 2] == '!'
+      && lexer->source.data[position + 3] == '>';
+}
+
+static inline bool lexer_peek_erb_start(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 1 < lexer->source.length && lexer->source.data[position] == '<'
+      && lexer->source.data[position + 1] == '%';
+}
+
+static inline bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 1 < lexer->source.length && lexer->source.data[position] == '%'
+      && lexer->source.data[position + 1] == '>';
+}
+
+static inline bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '-'
+      && lexer->source.data[position + 1] == '%' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '%'
+      && lexer->source.data[position + 1] == '%' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '='
+      && lexer->source.data[position + 1] == '%' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_erb_close_tag(lexer, offset) || lexer_peek_erb_dash_close_tag(lexer, offset)
+      || lexer_peek_erb_percent_close_tag(lexer, offset) || lexer_peek_erb_equals_close_tag(lexer, offset);
+}
+
+static inline lexer_state_snapshot_T lexer_save_state(lexer_T* lexer) {
+  lexer_state_snapshot_T snapshot = { .position = lexer->current_position,
+                                      .line = lexer->current_line,
+                                      .column = lexer->current_column,
+                                      .previous_position = lexer->previous_position,
+                                      .previous_line = lexer->previous_line,
+                                      .previous_column = lexer->previous_column,
+                                      .current_character = lexer->current_character,
+                                      .state = lexer->state };
+  return snapshot;
+}
 
-lexer_state_snapshot_T lexer_save_state(lexer_T* lexer);
-void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot);
+static inline void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot) {
+  lexer->current_position = snapshot.position;
+  lexer->current_line = snapshot.line;
+  lexer->current_column = snapshot.column;
+  lexer->previous_position = snapshot.previous_position;
+  lexer->previous_line = snapshot.previous_line;
+  lexer->previous_column = snapshot.previous_column;
+  lexer->current_character = snapshot.current_character;
+  lexer->state = snapshot.state;
+}
 
 #endif

data/src/include/lexer_struct.h

@@ -1,6 +1,7 @@
 #ifndef HERB_LEXER_STRUCT_H
 #define HERB_LEXER_STRUCT_H
 
+#include "util/hb_allocator.h"
 #include "util/hb_string.h"
 
 #include <stdbool.h>
@@ -14,6 +15,7 @@ typedef enum {
 } lexer_state_T;
 
 typedef struct LEXER_STRUCT {
+  hb_allocator_T* allocator;
   hb_string_T source;
 
   uint32_t current_line;

data/src/include/location.h

@@ -5,6 +5,7 @@
 #include <stdlib.h>
 
 #include "position.h"
+#include "util/hb_allocator.h"
 
 typedef struct LOCATION_STRUCT {
   position_T start;
@@ -21,6 +22,6 @@ void location_from(
 
 void location_from_positions(location_T* location, position_T start, position_T end);
 
-location_T* location_create(position_T start, position_T end);
+location_T* location_create(position_T start, position_T end, hb_allocator_T* allocator);
 
 #endif

data/src/include/parser.h

@@ -3,6 +3,7 @@
 
 #include "ast_node.h"
 #include "lexer.h"
+#include "util/hb_allocator.h"
 #include "util/hb_array.h"
 
 typedef enum {
@@ -17,17 +18,32 @@ typedef enum { PARSER_STATE_DATA, PARSER_STATE_FOREIGN_CONTENT } parser_state_T;
 
 typedef struct PARSER_OPTIONS_STRUCT {
   bool track_whitespace;
+  bool analyze;
+  bool strict;
+  bool action_view_helpers;
+  bool prism_program;
+  bool prism_nodes;
+  bool prism_nodes_deep;
 } parser_options_T;
 
+typedef struct MATCH_TAGS_CONTEXT_STRUCT {
+  hb_array_T* errors;
+  const parser_options_T* options;
+  hb_allocator_T* allocator;
+} match_tags_context_T;
+
 extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
 
 typedef struct PARSER_STRUCT {
+  hb_allocator_T* allocator;
   lexer_T* lexer;
   token_T* current_token;
   hb_array_T* open_tags_stack;
   parser_state_T state;
   foreign_content_type_T foreign_content_type;
   parser_options_T options;
+  size_t consecutive_error_count;
+  bool in_recovery_mode;
 } parser_T;
 
 size_t parser_sizeof(void);
@@ -36,10 +52,19 @@ void herb_parser_init(parser_T* parser, lexer_T* lexer, parser_options_T options
 
 AST_DOCUMENT_NODE_T* herb_parser_parse(parser_T* parser);
 
-void herb_parser_match_html_tags_post_analyze(AST_DOCUMENT_NODE_T* document);
+void herb_parser_match_html_tags_post_analyze(
+  AST_DOCUMENT_NODE_T* document,
+  const parser_options_T* options,
+  hb_allocator_T* allocator
+);
 void herb_parser_deinit(parser_T* parser);
 
-void match_tags_in_node_array(hb_array_T* nodes, hb_array_T* errors);
+void match_tags_in_node_array(
+  hb_array_T* nodes,
+  hb_array_T* errors,
+  const parser_options_T* options,
+  hb_allocator_T* allocator
+);
 bool match_tags_visitor(const AST_NODE_T* node, void* data);
 
 #endif

data/src/include/parser_helpers.h

@@ -13,11 +13,21 @@ void parser_push_open_tag(const parser_T* parser, token_T* tag_name);
 bool parser_check_matching_tag(const parser_T* parser, hb_string_T tag_name);
 token_T* parser_pop_open_tag(const parser_T* parser);
 
-void parser_append_unexpected_error(
+void parser_append_unexpected_error_impl(
   parser_T* parser,
+  hb_array_T* errors,
   const char* description,
-  const char* expected,
-  hb_array_T* errors
+  token_type_T first_token,
+  ...
+);
+#define parser_append_unexpected_error(parser, errors, description, ...)                                               \
+  parser_append_unexpected_error_impl(parser, errors, description, __VA_ARGS__, TOKEN_SENTINEL)
+
+void parser_append_unexpected_error_string(
+  parser_T* parser,
+  hb_array_T* errors,
+  const char* description,
+  const char* expected
 );
 void parser_append_unexpected_token_error(parser_T* parser, token_type_T expected_type, hb_array_T* errors);
 
@@ -44,6 +54,7 @@ token_T* parser_consume_if_present(parser_T* parser, token_type_T type);
 token_T* parser_consume_expected(parser_T* parser, token_type_T type, hb_array_T* array);
 
 AST_HTML_ELEMENT_NODE_T* parser_handle_missing_close_tag(
+  parser_T* parser,
   AST_HTML_OPEN_TAG_NODE_T* open_tag,
   hb_array_T* body,
   hb_array_T* errors
@@ -54,4 +65,9 @@ void parser_handle_mismatched_tags(
   hb_array_T* errors
 );
 
+void parser_synchronize(parser_T* parser, hb_array_T* errors);
+
+bool parser_can_close_ancestor(const parser_T* parser, hb_string_T tag_name);
+size_t parser_find_ancestor_depth(const parser_T* parser, hb_string_T tag_name);
+
 #endif

data/src/include/pretty_print.h

@@ -1,12 +1,16 @@
 #ifndef HERB_PRETTY_PRINT_H
 #define HERB_PRETTY_PRINT_H
 
-#include "analyzed_ruby.h"
-#include "ast_nodes.h"
-#include "location.h"
-#include "util/hb_buffer.h"
+#ifdef HERB_EXCLUDE_PRETTYPRINT
+// Pretty print support excluded
+#else
 
-#include <stdbool.h>
+#  include "analyze/analyzed_ruby.h"
+#  include "ast_nodes.h"
+#  include "location.h"
+#  include "util/hb_buffer.h"
+
+#  include <stdbool.h>
 
 void pretty_print_indent(hb_buffer_T* buffer, size_t indent);
 void pretty_print_newline(size_t indent, size_t relative_indent, hb_buffer_T* buffer);
@@ -101,3 +105,4 @@ void pretty_print_errors(
 );
 
 #endif
+#endif

data/src/include/prism_context.h

@@ -0,0 +1,45 @@
+#ifndef HERB_PRISM_CONTEXT_H
+#define HERB_PRISM_CONTEXT_H
+
+#include "util/hb_allocator.h"
+#include "util/hb_buffer.h"
+#include <prism.h>
+#include <stdbool.h>
+
+typedef struct {
+  pm_parser_t parser;
+  pm_options_t pm_opts;
+  pm_node_t* root;
+  hb_buffer_T ruby_buf;
+
+  bool has_structural;
+  pm_parser_t structural_parser;
+  pm_options_t structural_pm_opts;
+  pm_node_t* structural_root;
+  hb_buffer_T structural_buf;
+
+  hb_allocator_T* allocator;
+} herb_prism_context_T;
+
+static inline void herb_prism_context_free(herb_prism_context_T* context) {
+  if (!context) { return; }
+
+  if (context->root) { pm_node_destroy(&context->parser, context->root); }
+
+  pm_parser_free(&context->parser);
+  pm_options_free(&context->pm_opts);
+  hb_buffer_free(&context->ruby_buf);
+
+  if (context->has_structural) {
+    if (context->structural_root) { pm_node_destroy(&context->structural_parser, context->structural_root); }
+
+    pm_parser_free(&context->structural_parser);
+    pm_options_free(&context->structural_pm_opts);
+
+    if (context->structural_buf.value) { hb_buffer_free(&context->structural_buf); }
+  }
+
+  hb_allocator_dealloc(context->allocator, context);
+}
+
+#endif

data/src/include/prism_helpers.h

@@ -1,31 +1,34 @@
 #ifndef HERB_PRISM_HELPERS_H
 #define HERB_PRISM_HELPERS_H
 
-#include "analyzed_ruby.h"
+#include "analyze/analyzed_ruby.h"
 #include "ast_nodes.h"
 #include "errors.h"
 #include "location.h"
 #include "position.h"
+#include "util/hb_allocator.h"
 
 #include <prism.h>
 
-const char* pm_error_level_to_string(pm_error_level_t level);
+hb_string_T pm_error_level_to_string(pm_error_level_t level);
 
 RUBY_PARSE_ERROR_T* ruby_parse_error_from_prism_error(
   const pm_diagnostic_t* error,
   const AST_NODE_T* node,
   const char* source,
-  pm_parser_t* parser
+  pm_parser_t* parser,
+  hb_allocator_T* allocator
 );
 
 RUBY_PARSE_ERROR_T* ruby_parse_error_from_prism_error_with_positions(
   const pm_diagnostic_t* error,
   position_T start,
-  position_T end
+  position_T end,
+  hb_allocator_T* allocator
 );
 
-location_T* get_then_keyword_location(analyzed_ruby_T* analyzed, const char* source);
-location_T* get_then_keyword_location_wrapped(const char* source, bool is_in_clause);
-location_T* get_then_keyword_location_elsif_wrapped(const char* source);
+location_T* get_then_keyword_location(analyzed_ruby_T* analyzed, const char* source, hb_allocator_T* allocator);
+location_T* get_then_keyword_location_wrapped(const char* source, bool is_in_clause, hb_allocator_T* allocator);
+location_T* get_then_keyword_location_elsif_wrapped(const char* source, hb_allocator_T* allocator);
 
 #endif

data/src/include/prism_serialized.h

@@ -0,0 +1,12 @@
+#ifndef HERB_PRISM_SERIALIZED_H
+#define HERB_PRISM_SERIALIZED_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+  uint8_t* data;
+  size_t length;
+} prism_serialized_T;
+
+#endif

data/src/include/token.h

@@ -4,15 +4,27 @@
 #include "lexer_struct.h"
 #include "position.h"
 #include "token_struct.h"
+#include "util/hb_allocator.h"
 #include "util/hb_string.h"
 
+#include <stdarg.h>
+
 token_T* token_init(hb_string_T value, token_type_T type, lexer_T* lexer);
-hb_string_T token_to_string(const token_T* token);
-const char* token_type_to_string(token_type_T type);
+hb_string_T token_to_string(hb_allocator_T* allocator, const token_T* token);
+hb_string_T token_type_to_string(token_type_T type);
+hb_string_T token_type_to_friendly_string(token_type_T type);
+char* token_types_to_friendly_string_va(hb_allocator_T* allocator, token_type_T first_token, ...);
+char* token_types_to_friendly_string_valist(hb_allocator_T* allocator, token_type_T first_token, va_list args);
+
+#define token_types_to_friendly_string(allocator, ...)                                                                 \
+  token_types_to_friendly_string_va(allocator, __VA_ARGS__, TOKEN_SENTINEL)
+
+hb_string_T token_value(const token_T* token);
+int token_type(const token_T* token);
 
-token_T* token_copy(token_T* token);
+token_T* token_copy(token_T* token, hb_allocator_T* allocator);
 
-void token_free(token_T* token);
+void token_free(token_T* token, hb_allocator_T* allocator);
 
 bool token_value_empty(const token_T* token);
 

data/src/include/token_struct.h

@@ -1,8 +1,11 @@
 #ifndef HERB_TOKEN_STRUCT_H
 #define HERB_TOKEN_STRUCT_H
 
+#include <stdbool.h>
+
 #include "location.h"
 #include "range.h"
+#include "util/hb_string.h"
 
 typedef enum {
   TOKEN_WHITESPACE, // ' '
@@ -21,8 +24,9 @@ typedef enum {
   TOKEN_HTML_TAG_END,         // >
   TOKEN_HTML_TAG_SELF_CLOSE,  // />
 
-  TOKEN_HTML_COMMENT_START, // <!--
-  TOKEN_HTML_COMMENT_END,   // -->
+  TOKEN_HTML_COMMENT_START,       // <!--
+  TOKEN_HTML_COMMENT_END,         // -->
+  TOKEN_HTML_COMMENT_INVALID_END, // --!>
 
   TOKEN_ERB_START,   // <%, <%=, <%%=, <%#, <%-, <%==, <%%
   TOKEN_ERB_CONTENT, // Ruby Code
@@ -48,8 +52,11 @@ typedef enum {
   TOKEN_EOF,
 } token_type_T;
 
+// Sentinel value for variadic functions
+#define TOKEN_SENTINEL 99999999
+
 typedef struct TOKEN_STRUCT {
-  char* value;
+  hb_string_T value;
   range_T range;
   location_T location;
   token_type_T type;

data/src/include/utf8.h

@@ -1,12 +1,13 @@
 #ifndef HERB_UTF8_H
 #define HERB_UTF8_H
 
+#include "util/hb_string.h"
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdlib.h>
 
 uint32_t utf8_char_byte_length(unsigned char first_byte);
-uint32_t utf8_sequence_length(const char* str, size_t position, size_t max_length);
+uint32_t utf8_sequence_length(hb_string_T value);
 bool utf8_is_valid_continuation_byte(unsigned char byte);
 
 #endif