herb 0.10.3-aarch64-linux-gnu → 0.11.0-aarch64-linux-gnu

data/src/include/lexer/lexer_peek_helpers.h

@@ -21,10 +21,13 @@ typedef struct {
   uint32_t previous_column;
   char current_character;
   lexer_state_T state;
+  bool in_erb_comment;
+  uint8_t malformed_erb_close_length;
 } lexer_state_snapshot_T;
 
 bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_xml_processing_instruction(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_cdata_end(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_html_comment_start(const lexer_T* lexer, uint32_t offset);
@@ -102,7 +105,9 @@ static inline lexer_state_snapshot_T lexer_save_state(lexer_T* lexer) {
                                       .previous_line = lexer->previous_line,
                                       .previous_column = lexer->previous_column,
                                       .current_character = lexer->current_character,
-                                      .state = lexer->state };
+                                      .state = lexer->state,
+                                      .in_erb_comment = lexer->in_erb_comment,
+                                      .malformed_erb_close_length = lexer->malformed_erb_close_length };
   return snapshot;
 }
 
@@ -115,6 +120,8 @@ static inline void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T sn
   lexer->previous_column = snapshot.previous_column;
   lexer->current_character = snapshot.current_character;
   lexer->state = snapshot.state;
+  lexer->in_erb_comment = snapshot.in_erb_comment;
+  lexer->malformed_erb_close_length = snapshot.malformed_erb_close_length;
 }
 
 #endif

data/src/include/lexer/lexer_struct.h

@@ -3,6 +3,7 @@
 
 #include "../lib/hb_allocator.h"
 #include "../lib/hb_string.h"
+#include "erb_openers.h"
 
 #include <stdbool.h>
 #include <stdint.h>
@@ -28,6 +29,9 @@ typedef struct LEXER_STRUCT {
 
   char current_character;
   lexer_state_T state;
+  erb_openers_T erb_openers;
+  bool in_erb_comment;
+  uint8_t malformed_erb_close_length;
   uint32_t stall_counter;
   uint32_t last_position;
   bool stalled;

data/src/include/lexer/token.h

@@ -8,6 +8,8 @@
 #include "token_struct.h"
 
 #include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
 
 token_T* token_init(hb_string_T value, token_type_T type, lexer_T* lexer);
 hb_string_T token_to_string(hb_allocator_T* allocator, const token_T* token);
@@ -23,6 +25,7 @@ hb_string_T token_value(const token_T* token);
 int token_type(const token_T* token);
 
 token_T* token_copy(token_T* token, hb_allocator_T* allocator);
+bool token_split(const token_T* token, uint32_t offset, hb_allocator_T* allocator, token_T** head, token_T** tail);
 
 void token_free(token_T* token, hb_allocator_T* allocator);
 

data/src/include/lexer/token_struct.h

@@ -13,11 +13,12 @@ typedef enum {
   TOKEN_NEWLINE,    // \n
   TOKEN_IDENTIFIER,
 
-  TOKEN_HTML_DOCTYPE,        // <!DOCTYPE, <!doctype, <!DoCtYpE, <!dOcTyPe
-  TOKEN_XML_DECLARATION,     // <?xml
-  TOKEN_XML_DECLARATION_END, // ?>
-  TOKEN_CDATA_START,         // <![CDATA[
-  TOKEN_CDATA_END,           // ]]>
+  TOKEN_HTML_DOCTYPE,                     // <!DOCTYPE, <!doctype, <!DoCtYpE, <!dOcTyPe
+  TOKEN_XML_DECLARATION,                  // <?xml
+  TOKEN_XML_DECLARATION_END,              // ?>
+  TOKEN_XML_PROCESSING_INSTRUCTION_START, // <?
+  TOKEN_CDATA_START,                      // <![CDATA[
+  TOKEN_CDATA_END,                        // ]]>
 
   TOKEN_HTML_TAG_START,       // <
   TOKEN_HTML_TAG_START_CLOSE, // </
@@ -30,7 +31,7 @@ typedef enum {
 
   TOKEN_ERB_START,   // <%, <%=, <%%=, <%#, <%-, <%==, <%%
   TOKEN_ERB_CONTENT, // Ruby Code
-  TOKEN_ERB_END,     // %>, -%>, %%>
+  TOKEN_ERB_END,     // %>, -%>, =%>, %%>, and recovered: >, %, -%, =%, %%, "% >"
 
   TOKEN_LT,          // <
   TOKEN_SLASH,       // /
@@ -60,6 +61,7 @@ typedef struct TOKEN_STRUCT {
   range_T range;
   location_T location;
   token_type_T type;
+  bool owns_value;
 } token_T;
 
 #endif

data/src/include/lib/hb_allocator.h

@@ -4,10 +4,11 @@
 #include <stdbool.h>
 #include <stddef.h>
 
+#include "../macros.h"
 #include "hb_arena.h"
 
 #ifndef HB_ALLOCATOR_DEFAULT_ARENA_SIZE
-#  define HB_ALLOCATOR_DEFAULT_ARENA_SIZE (1024 * 16)
+#  define HB_ALLOCATOR_DEFAULT_ARENA_SIZE KB(128)
 #endif
 
 typedef enum {

data/src/include/lib/hb_array.h

@@ -20,8 +20,10 @@ void* hb_array_first(hb_array_T* array);
 void* hb_array_last(hb_array_T* array);
 
 bool hb_array_append(hb_array_T* array, void* item);
+bool hb_array_append_lazy(hb_array_T** array, void* item, struct hb_allocator* allocator);
 void hb_array_set(const hb_array_T* array, size_t index, void* item);
 void hb_array_free(hb_array_T** array);
+void hb_array_replace_contents(hb_array_T* array, hb_array_T** source);
 void hb_array_remove(hb_array_T* array, size_t index);
 
 size_t hb_array_index_of(hb_array_T* array, void* item);

data/src/include/lib/hb_foreach.h

@@ -1,5 +1,5 @@
 // NOTE: This file is generated by the templates/template.rb script and should not
-// be modified manually. See /home/runner/work/herb/herb/templates/src/include/lib/hb_foreach.h.erb
+// be modified manually. See templates/src/include/lib/hb_foreach.h.erb
 
 #ifndef HB_FOREACH_H
 #define HB_FOREACH_H

data/src/include/lib/hb_string.h

@@ -59,6 +59,12 @@ static inline bool hb_string_equals_case_insensitive(hb_string_T a, hb_string_T
   return strncasecmp(a.data, b.data, a.length) == 0;
 }
 
+static inline bool hb_string_contains_character(hb_string_T string, char character) {
+  if (hb_string_is_empty(string)) { return false; }
+
+  return memchr(string.data, character, string.length) != NULL;
+}
+
 static inline bool hb_string_starts_with(hb_string_T string, hb_string_T expected_prefix) {
   if (hb_string_is_empty(string) || hb_string_is_empty(expected_prefix)) { return false; }
   if (string.length < expected_prefix.length) { return false; }
@@ -72,6 +78,7 @@ hb_string_T hb_string_trim_start(hb_string_T string);
 hb_string_T hb_string_trim_end(hb_string_T string);
 hb_string_T hb_string_trim(hb_string_T string);
 bool hb_string_is_blank(hb_string_T string);
+bool hb_string_equals_collapsing_whitespace(hb_string_T a, hb_string_T b);
 hb_string_T hb_string_copy(hb_string_T string, hb_allocator_T* allocator);
 
 char* hb_string_to_c_string_using_malloc(hb_string_T string);

data/src/include/parser/dot_notation.h

@@ -7,6 +7,6 @@
 #include "parser.h"
 
 bool parser_lookahead_is_valid_dot_notation_open_tag(parser_T* parser);
-void parser_consume_dot_notation_segments(parser_T* parser, token_T* tag_name, hb_array_T* errors);
+void parser_consume_dot_notation_segments(parser_T* parser, token_T* tag_name, hb_array_T** errors);
 
 #endif

data/src/include/parser/foreign_content_elements.h

@@ -0,0 +1,32 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See templates/src/include/parser/foreign_content_elements.h.erb
+
+#ifndef HERB_FOREIGN_CONTENT_ELEMENTS_H
+#define HERB_FOREIGN_CONTENT_ELEMENTS_H
+
+#include "parser.h"
+
+#include <stdbool.h>
+
+typedef struct {
+  const char* name;
+  foreign_content_kind_T kind;
+  bool has_end_tag;
+  bool html_only;
+} foreign_content_element_T;
+
+static const foreign_content_element_T FOREIGN_CONTENT_ELEMENTS[] = {
+  { "script", FOREIGN_CONTENT_RAW_TEXT, true, false },
+  { "style", FOREIGN_CONTENT_RAW_TEXT, true, false },
+  { "iframe", FOREIGN_CONTENT_RAW_TEXT, true, false },
+  { "xmp", FOREIGN_CONTENT_RAW_TEXT, true, false },
+  { "noembed", FOREIGN_CONTENT_RAW_TEXT, true, false },
+  { "noframes", FOREIGN_CONTENT_RAW_TEXT, true, false },
+  { "plaintext", FOREIGN_CONTENT_RAW_TEXT, false, true },
+  { "textarea", FOREIGN_CONTENT_RCDATA, true, false },
+  { "title", FOREIGN_CONTENT_RCDATA, true, true },
+};
+
+#define FOREIGN_CONTENT_ELEMENTS_COUNT (sizeof(FOREIGN_CONTENT_ELEMENTS) / sizeof(FOREIGN_CONTENT_ELEMENTS[0]))
+
+#endif

data/src/include/parser/parser.h

@@ -6,47 +6,29 @@
 #include "../lib/hb_allocator.h"
 #include "../lib/hb_array.h"
 #include "../lib/hb_clock.h"
+#include "parser_options.h"
 
-#include <stdint.h>
+#define HERB_MAX_FOREIGN_CONTENT_ELEMENTS 16
 
 typedef enum {
-  FOREIGN_CONTENT_UNKNOWN = 0,
-  FOREIGN_CONTENT_SCRIPT,
-  FOREIGN_CONTENT_STYLE,
-  // FOREIGN_CONTENT_RUBY,
-  // FOREIGN_CONTENT_TEMPLATE
-} foreign_content_type_T;
+  FOREIGN_CONTENT_NONE = 0,
+  FOREIGN_CONTENT_RAW_TEXT,
+  FOREIGN_CONTENT_RCDATA,
+} foreign_content_kind_T;
 
 typedef enum { PARSER_STATE_DATA, PARSER_STATE_FOREIGN_CONTENT } parser_state_T;
 
-typedef struct PARSER_OPTIONS_STRUCT {
-  bool track_whitespace;
-  bool analyze;
-  bool strict;
-  bool action_view_helpers;
-  bool transform_conditionals;
-  bool render_nodes;
-  bool strict_locals;
-  bool prism_program;
-  bool prism_nodes;
-  bool prism_nodes_deep;
-  bool dot_notation_tags;
-  bool html;
-  uint32_t start_line;
-  uint32_t start_column;
-  uint32_t timeout_ms;
-  uint32_t max_errors;
-  uint32_t* error_count;
-  uint64_t deadline_ms;
-} parser_options_T;
-
 typedef struct MATCH_TAGS_CONTEXT_STRUCT {
   hb_array_T* errors;
   const parser_options_T* options;
   hb_allocator_T* allocator;
 } match_tags_context_T;
 
-extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
+static inline void lexer_apply_erb_openers(lexer_T* lexer, const parser_options_T* options) {
+  if (options == NULL || options->erb_openers == NULL) { return; }
+
+  lexer->erb_openers = (erb_openers_T) { .items = options->erb_openers, .count = options->erb_opener_count };
+}
 
 static inline bool parser_options_past_deadline(const parser_options_T* options) {
   if (options == NULL || options->timeout_ms == 0) { return false; }
@@ -77,7 +59,11 @@ typedef struct PARSER_STRUCT {
   token_T* current_token;
   hb_array_T* open_tags_stack;
   parser_state_T state;
-  foreign_content_type_T foreign_content_type;
+  foreign_content_kind_T foreign_content_kind;
+  hb_string_T foreign_content_tag_name;
+  size_t svg_depth;
+  bool xml_document;
+  uint32_t foreign_content_absent_from[HERB_MAX_FOREIGN_CONTENT_ELEMENTS];
   parser_options_T options;
   size_t consecutive_error_count;
   bool in_recovery_mode;

data/src/include/parser/parser_helpers.h

@@ -15,7 +15,7 @@ token_T* parser_pop_open_tag(const parser_T* parser);
 
 void parser_append_unexpected_error_impl(
   parser_T* parser,
-  hb_array_T* errors,
+  hb_array_T** errors,
   const char* description,
   token_type_T first_token,
   ...
@@ -25,11 +25,11 @@ void parser_append_unexpected_error_impl(
 
 void parser_append_unexpected_error_string(
   parser_T* parser,
-  hb_array_T* errors,
+  hb_array_T** errors,
   const char* description,
   const char* expected
 );
-void parser_append_unexpected_token_error(parser_T* parser, token_type_T expected_type, hb_array_T* errors);
+void parser_append_unexpected_token_error(parser_T* parser, token_type_T expected_type, hb_array_T** errors);
 
 void parser_append_literal_node_from_buffer(
   const parser_T* parser,
@@ -40,32 +40,34 @@ void parser_append_literal_node_from_buffer(
 
 bool parser_in_svg_context(const parser_T* parser);
 
-foreign_content_type_T parser_get_foreign_content_type(hb_string_T tag_name);
+foreign_content_kind_T parser_get_foreign_content_kind(hb_string_T tag_name);
 bool parser_is_foreign_content_tag(hb_string_T tag_name);
-hb_string_T parser_get_foreign_content_closing_tag(foreign_content_type_T type);
+bool parser_foreign_content_has_end_tag(hb_string_T tag_name);
+bool parser_foreign_content_is_html_only(hb_string_T tag_name);
+int parser_foreign_content_element_index(hb_string_T tag_name);
 
-void parser_enter_foreign_content(parser_T* parser, foreign_content_type_T type);
+void parser_enter_foreign_content(parser_T* parser, foreign_content_kind_T kind, hb_string_T tag_name);
 void parser_exit_foreign_content(parser_T* parser);
 
-bool parser_is_expected_closing_tag_name(hb_string_T tag_name, foreign_content_type_T expected_type);
+bool parser_is_foreign_content_closing_tag_name(const parser_T* parser, hb_string_T tag_name);
 
 token_T* parser_advance(parser_T* parser);
 token_T* parser_consume_if_present(parser_T* parser, token_type_T type);
-token_T* parser_consume_expected(parser_T* parser, token_type_T type, hb_array_T* array);
+token_T* parser_consume_expected(parser_T* parser, token_type_T type, hb_array_T** array);
 
 AST_HTML_ELEMENT_NODE_T* parser_handle_missing_close_tag(
   parser_T* parser,
   AST_HTML_OPEN_TAG_NODE_T* open_tag,
   hb_array_T* body,
-  hb_array_T* errors
+  hb_array_T** errors
 );
 void parser_handle_mismatched_tags(
   const parser_T* parser,
   const AST_HTML_CLOSE_TAG_NODE_T* close_tag,
-  hb_array_T* errors
+  hb_array_T** errors
 );
 
-void parser_synchronize(parser_T* parser, hb_array_T* errors);
+void parser_synchronize(parser_T* parser, hb_array_T** errors);
 
 bool parser_can_close_ancestor(const parser_T* parser, hb_string_T tag_name);
 size_t parser_find_ancestor_depth(const parser_T* parser, hb_string_T tag_name);

data/src/include/parser/parser_options.h

@@ -0,0 +1,41 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See templates/src/include/parser/parser_options.h.erb
+
+#ifndef HERB_PARSER_OPTIONS_H
+#define HERB_PARSER_OPTIONS_H
+
+#include "../lib/hb_string.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct PARSER_OPTIONS_STRUCT {
+  bool action_view_helpers;
+  bool analyze;
+  bool dot_notation_tags;
+  const hb_string_T* erb_openers;
+  size_t erb_opener_count;
+  bool herb_directives;
+  bool html;
+  bool iteration_nodes;
+  uint32_t max_errors;
+  bool prism_nodes;
+  bool prism_nodes_deep;
+  bool prism_program;
+  bool render_nodes;
+  bool strict;
+  bool strict_locals;
+  uint32_t timeout_ms;
+  bool track_locations;
+  bool track_whitespace;
+  bool transform_conditionals;
+  uint64_t deadline_ms;
+  uint32_t* error_count;
+  uint32_t start_column;
+  uint32_t start_line;
+} parser_options_T;
+
+extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
+
+#endif

data/src/include/prism/ruby_parser.h

@@ -1,6 +1,12 @@
 #ifndef HERB_RUBY_PARSER_H
 #define HERB_RUBY_PARSER_H
 
+#include "../lib/hb_string.h"
+
+#include <stdbool.h>
+
 void herb_parse_ruby_to_stdout(char* source);
 
+bool herb_ruby_fragment_is_parseable(hb_string_T source);
+
 #endif

data/src/include/util/html_elements.h

@@ -0,0 +1,85 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See templates/src/include/util/html_elements.h.erb
+
+#ifndef HERB_HTML_ELEMENTS_H
+#define HERB_HTML_ELEMENTS_H
+
+#include "../lib/hb_string.h"
+
+// https://developer.mozilla.org/en-US/docs/Glossary/Void_element
+static hb_string_T void_tags[] = HB_STRING_LIST(
+  "area",
+  "base",
+  "basefont",
+  "bgsound",
+  "br",
+  "col",
+  "embed",
+  "frame",
+  "hr",
+  "img",
+  "input",
+  "keygen",
+  "link",
+  "meta",
+  "param",
+  "source",
+  "track",
+  "wbr"
+);
+
+// https://html.spec.whatwg.org/multipage/common-microsyntaxes.html#boolean-attributes
+static hb_string_T boolean_attributes[] = HB_STRING_LIST(
+  "allowfullscreen",
+  "async",
+  "autofocus",
+  "autoplay",
+  "checked",
+  "compact",
+  "controls",
+  "declare",
+  "default",
+  "defer",
+  "disabled",
+  "formnovalidate",
+  "hidden",
+  "inert",
+  "ismap",
+  "itemscope",
+  "loop",
+  "multiple",
+  "muted",
+  "nomodule",
+  "nohref",
+  "noresize",
+  "noshade",
+  "novalidate",
+  "nowrap",
+  "open",
+  "playsinline",
+  "readonly",
+  "required",
+  "reversed",
+  "scoped",
+  "seamless",
+  "selected",
+  "sortable",
+  "truespeed",
+  "typemustmatch"
+);
+
+// https://html.spec.whatwg.org/multipage/rendering.html#the-page
+static hb_string_T whitespace_preserving_tags[] = HB_STRING_LIST(
+  "iframe",
+  "listing",
+  "noembed",
+  "noframes",
+  "plaintext",
+  "pre",
+  "script",
+  "style",
+  "textarea",
+  "xmp"
+);
+
+#endif

data/src/include/util/html_util.h

@@ -7,6 +7,7 @@
 struct hb_allocator;
 
 bool is_void_element(hb_string_T tag_name);
+bool is_whitespace_preserving_element(hb_string_T tag_name);
 bool is_boolean_attribute(hb_string_T attribute_name);
 bool has_optional_end_tag(hb_string_T tag_name);
 bool should_implicitly_close(hb_string_T open_tag_name, hb_string_T next_tag_name);

data/src/include/version.h

@@ -1,6 +1,6 @@
 #ifndef HERB_VERSION_H
 #define HERB_VERSION_H
 
-#define HERB_VERSION "0.10.3"
+#define HERB_VERSION "0.11.0"
 
 #endif

data/src/lexer/lexer_peek_helpers.c

@@ -19,8 +19,22 @@ bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset) {
   return lexer_peek_for(lexer, offset, hb_string("<!DOCTYPE"), true);
 }
 
+static bool is_xml_name_character(char character) {
+  return isalnum(character) || character == '-' || character == '_' || character == ':' || character == '.';
+}
+
 bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("<?xml"), true);
+  hb_string_T pattern = hb_string("<?xml");
+
+  if (!lexer_peek_for(lexer, offset, pattern, true)) { return false; }
+
+  return !is_xml_name_character(lexer_peek(lexer, offset + pattern.length));
+}
+
+bool lexer_peek_for_xml_processing_instruction(const lexer_T* lexer, uint32_t offset) {
+  if (lexer_peek(lexer, offset) != '<' || lexer_peek(lexer, offset + 1) != '?') { return false; }
+
+  return isalpha(lexer_peek(lexer, offset + 2));
 }
 
 bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset) {

data/src/lexer/token.c

@@ -5,6 +5,7 @@
 #include "../include/lib/hb_string.h"
 #include "../include/location/position.h"
 #include "../include/location/range.h"
+#include "../include/util/utf8.h"
 #include "../include/util/util.h"
 
 #include <stdarg.h>
@@ -25,6 +26,7 @@ token_T* token_init(hb_string_T value, const token_type_T type, lexer_T* lexer)
   }
 
   token->value = value;
+  token->owns_value = false;
 
   token->type = type;
   token->range = (range_T) { .from = lexer->previous_position, .to = lexer->current_position };
@@ -53,6 +55,7 @@ hb_string_T token_type_to_string(const token_type_T type) {
     case TOKEN_HTML_DOCTYPE: return hb_string("TOKEN_HTML_DOCTYPE");
     case TOKEN_XML_DECLARATION: return hb_string("TOKEN_XML_DECLARATION");
     case TOKEN_XML_DECLARATION_END: return hb_string("TOKEN_XML_DECLARATION_END");
+    case TOKEN_XML_PROCESSING_INSTRUCTION_START: return hb_string("TOKEN_XML_PROCESSING_INSTRUCTION_START");
     case TOKEN_CDATA_START: return hb_string("TOKEN_CDATA_START");
     case TOKEN_CDATA_END: return hb_string("TOKEN_CDATA_END");
     case TOKEN_HTML_TAG_START: return hb_string("TOKEN_HTML_TAG_START");
@@ -94,6 +97,7 @@ hb_string_T token_type_to_friendly_string(const token_type_T type) {
     case TOKEN_HTML_DOCTYPE: return hb_string("`<!DOCTYPE`");
     case TOKEN_XML_DECLARATION: return hb_string("`<?xml`");
     case TOKEN_XML_DECLARATION_END: return hb_string("`?>`");
+    case TOKEN_XML_PROCESSING_INSTRUCTION_START: return hb_string("`<?`");
     case TOKEN_CDATA_START: return hb_string("`<![CDATA[`");
     case TOKEN_CDATA_END: return hb_string("`]]>`");
     case TOKEN_HTML_TAG_START: return hb_string("`<`");
@@ -215,7 +219,8 @@ token_T* token_copy(token_T* token, hb_allocator_T* allocator) {
 
   if (!new_token) { return NULL; }
 
-  new_token->value = token->value;
+  new_token->value = token->owns_value ? hb_string_copy(token->value, allocator) : token->value;
+  new_token->owns_value = token->owns_value;
 
   new_token->type = token->type;
   new_token->range = token->range;
@@ -224,6 +229,81 @@ token_T* token_copy(token_T* token, hb_allocator_T* allocator) {
   return new_token;
 }
 
+static position_T token_position_after(position_T position, hb_string_T value, uint32_t offset) {
+  uint32_t index = 0;
+
+  while (index < offset && index < value.length) {
+    if (is_newline(value.data[index])) {
+      position.line++;
+      position.column = 0;
+      index++;
+
+      continue;
+    }
+
+    position.column++;
+    index += utf8_sequence_length(hb_string_slice(value, index));
+  }
+
+  return position;
+}
+
+static token_T* token_from_slice(
+  const token_T* token,
+  hb_string_T value,
+  uint32_t offset,
+  position_T start,
+  position_T end,
+  hb_allocator_T* allocator
+) {
+  token_T* slice = hb_allocator_alloc(allocator, sizeof(token_T));
+
+  if (!slice) { return NULL; }
+
+  slice->value = token->owns_value ? hb_string_copy(value, allocator) : value;
+  slice->owns_value = token->owns_value;
+
+  slice->type = token->type;
+  slice->range = (range_T) { .from = token->range.from + offset, .to = token->range.from + offset + value.length };
+
+  location_from_positions(&slice->location, start, end);
+
+  return slice;
+}
+
+bool token_split(
+  const token_T* token,
+  const uint32_t offset,
+  hb_allocator_T* allocator,
+  token_T** head,
+  token_T** tail
+) {
+  if (!token || !head || !tail) { return false; }
+  if (offset == 0 || offset >= token->value.length) { return false; }
+  if (utf8_is_valid_continuation_byte((unsigned char) token->value.data[offset])) { return false; }
+
+  const position_T split = token_position_after(token->location.start, token->value, offset);
+
+  token_T* head_token =
+    token_from_slice(token, hb_string_range(token->value, 0, offset), 0, token->location.start, split, allocator);
+
+  if (!head_token) { return false; }
+
+  token_T* tail_token =
+    token_from_slice(token, hb_string_slice(token->value, offset), offset, split, token->location.end, allocator);
+
+  if (!tail_token) {
+    token_free(head_token, allocator);
+
+    return false;
+  }
+
+  *head = head_token;
+  *tail = tail_token;
+
+  return true;
+}
+
 bool token_value_empty(const token_T* token) {
   return token == NULL || hb_string_is_empty(token->value);
 }
@@ -237,5 +317,7 @@ bool token_is_escaped_erb_tag_opening(const token_T* token) {
 void token_free(token_T* token, hb_allocator_T* allocator) {
   if (!token) { return; }
 
+  if (token->owns_value) { hb_allocator_dealloc(allocator, (void*) token->value.data); }
+
   hb_allocator_dealloc(allocator, token);
 }