herb 0.5.0 → 0.6.0

data/src/parser.c

@@ -6,6 +6,7 @@
 #include "include/errors.h"
 #include "include/html_util.h"
 #include "include/lexer.h"
+#include "include/lexer_peek_helpers.h"
 #include "include/parser_helpers.h"
 #include "include/token.h"
 #include "include/token_matchers.h"
@@ -19,12 +20,18 @@
 static void parser_parse_in_data_state(parser_T* parser, array_T* children, array_T* errors);
 static void parser_parse_foreign_content(parser_T* parser, array_T* children, array_T* errors);
 static AST_ERB_CONTENT_NODE_T* parser_parse_erb_tag(parser_T* parser);
+static void parser_handle_whitespace(parser_T* parser, token_T* whitespace_token, array_T* children);
+static void parser_consume_whitespace(parser_T* parser, array_T* children);
+static void parser_skip_erb_content(lexer_T* lexer);
+static bool parser_lookahead_erb_is_attribute(lexer_T* lexer);
+static void parser_handle_erb_in_open_tag(parser_T* parser, array_T* children);
+static void parser_handle_whitespace_in_open_tag(parser_T* parser, array_T* children);
 
 size_t parser_sizeof(void) {
   return sizeof(struct PARSER_STRUCT);
 }
 
-parser_T* parser_init(lexer_T* lexer) {
+parser_T* parser_init(lexer_T* lexer, parser_options_T* options) {
   parser_T* parser = calloc(1, parser_sizeof());
 
   parser->lexer = lexer;
@@ -33,9 +40,59 @@ parser_T* parser_init(lexer_T* lexer) {
   parser->state = PARSER_STATE_DATA;
   parser->foreign_content_type = FOREIGN_CONTENT_UNKNOWN;
 
+  if (options) {
+    parser->options = calloc(1, sizeof(parser_options_T));
+    parser->options->track_whitespace = options->track_whitespace;
+  } else {
+    parser->options = NULL;
+  }
+
   return parser;
 }
 
+static AST_CDATA_NODE_T* parser_parse_cdata(parser_T* parser) {
+  array_T* errors = array_init(8);
+  array_T* children = array_init(8);
+  buffer_T content = buffer_new();
+
+  token_T* tag_opening = parser_consume_expected(parser, TOKEN_CDATA_START, errors);
+  position_T* start = position_copy(parser->current_token->location->start);
+
+  while (token_is_none_of(parser, TOKEN_CDATA_END, TOKEN_EOF)) {
+    if (token_is(parser, TOKEN_ERB_START)) {
+      parser_append_literal_node_from_buffer(parser, &content, children, start);
+      AST_ERB_CONTENT_NODE_T* erb_node = parser_parse_erb_tag(parser);
+      array_append(children, erb_node);
+      position_free(start);
+      start = position_copy(parser->current_token->location->start);
+      continue;
+    }
+
+    token_T* token = parser_advance(parser);
+    buffer_append(&content, token->value);
+    token_free(token);
+  }
+
+  parser_append_literal_node_from_buffer(parser, &content, children, start);
+  token_T* tag_closing = parser_consume_expected(parser, TOKEN_CDATA_END, errors);
+
+  AST_CDATA_NODE_T* cdata = ast_cdata_node_init(
+    tag_opening,
+    children,
+    tag_closing,
+    tag_opening->location->start,
+    tag_closing->location->end,
+    errors
+  );
+
+  position_free(start);
+  buffer_free(&content);
+  token_free(tag_opening);
+  token_free(tag_closing);
+
+  return cdata;
+}
+
 static AST_HTML_COMMENT_NODE_T* parser_parse_html_comment(parser_T* parser) {
   array_T* errors = array_init(8);
   array_T* children = array_init(8);
@@ -128,6 +185,54 @@ static AST_HTML_DOCTYPE_NODE_T* parser_parse_html_doctype(parser_T* parser) {
   return doctype;
 }
 
+static AST_XML_DECLARATION_NODE_T* parser_parse_xml_declaration(parser_T* parser) {
+  array_T* errors = array_init(8);
+  array_T* children = array_init(8);
+  buffer_T content = buffer_new();
+
+  token_T* tag_opening = parser_consume_expected(parser, TOKEN_XML_DECLARATION, errors);
+
+  position_T* start = position_copy(parser->current_token->location->start);
+
+  while (token_is_none_of(parser, TOKEN_XML_DECLARATION_END, TOKEN_EOF)) {
+    if (token_is(parser, TOKEN_ERB_START)) {
+      parser_append_literal_node_from_buffer(parser, &content, children, start);
+
+      AST_ERB_CONTENT_NODE_T* erb_node = parser_parse_erb_tag(parser);
+      array_append(children, erb_node);
+
+      position_free(start);
+      start = position_copy(parser->current_token->location->start);
+
+      continue;
+    }
+
+    token_T* token = parser_advance(parser);
+    buffer_append(&content, token->value);
+    token_free(token);
+  }
+
+  parser_append_literal_node_from_buffer(parser, &content, children, start);
+
+  token_T* tag_closing = parser_consume_expected(parser, TOKEN_XML_DECLARATION_END, errors);
+
+  AST_XML_DECLARATION_NODE_T* xml_declaration = ast_xml_declaration_node_init(
+    tag_opening,
+    children,
+    tag_closing,
+    tag_opening->location->start,
+    tag_closing->location->end,
+    errors
+  );
+
+  position_free(start);
+  token_free(tag_opening);
+  token_free(tag_closing);
+  buffer_free(&content);
+
+  return xml_declaration;
+}
+
 static AST_HTML_TEXT_NODE_T* parser_parse_text_content(parser_T* parser, array_T* document_errors) {
   position_T* start = position_copy(parser->current_token->location->start);
 
@@ -188,96 +293,58 @@ static AST_HTML_TEXT_NODE_T* parser_parse_text_content(parser_T* parser, array_T
 
 static AST_HTML_ATTRIBUTE_NAME_NODE_T* parser_parse_html_attribute_name(parser_T* parser) {
   array_T* errors = array_init(8);
+  array_T* children = array_init(8);
+  buffer_T buffer = buffer_new();
+  position_T* start = position_copy(parser->current_token->location->start);
 
-  token_T* at_token = parser_consume_if_present(parser, TOKEN_AT);
-  token_T* first_token = NULL;
-
-  if (at_token != NULL) {
-    first_token = parser_consume_if_present(parser, TOKEN_IDENTIFIER);
-
-    if (first_token == NULL) {
-      parser_append_unexpected_token_error(parser, TOKEN_IDENTIFIER, errors);
-
-      AST_HTML_ATTRIBUTE_NAME_NODE_T* attribute_name =
-        ast_html_attribute_name_node_init(at_token, at_token->location->start, at_token->location->end, errors);
-
-      token_free(at_token);
+  while (token_is_none_of(
+    parser,
+    TOKEN_EQUALS,
+    TOKEN_WHITESPACE,
+    TOKEN_NEWLINE,
+    TOKEN_HTML_TAG_END,
+    TOKEN_HTML_TAG_SELF_CLOSE,
+    TOKEN_EOF
+  )) {
+    if (token_is(parser, TOKEN_ERB_START)) {
+      parser_append_literal_node_from_buffer(parser, &buffer, children, start);
 
-      return attribute_name;
-    }
-  } else {
-    first_token = parser_consume_if_present(parser, TOKEN_IDENTIFIER);
+      AST_ERB_CONTENT_NODE_T* erb_node = parser_parse_erb_tag(parser);
+      array_append(children, erb_node);
 
-    if (first_token == NULL) {
-      parser_append_unexpected_token_error(parser, TOKEN_IDENTIFIER, errors);
-      AST_HTML_ATTRIBUTE_NAME_NODE_T* attribute_name = ast_html_attribute_name_node_init(NULL, NULL, NULL, errors);
-      return attribute_name;
+      position_free(start);
+      start = position_copy(parser->current_token->location->start);
+      continue;
     }
-  }
-
-  buffer_T name_buffer = buffer_new();
 
-  position_T* start_position;
-
-  if (at_token != NULL) {
-    buffer_append(&name_buffer, at_token->value);
-    start_position = position_copy(at_token->location->start);
-  } else {
-    start_position = position_copy(first_token->location->start);
+    token_T* token = parser_advance(parser);
+    buffer_append(&buffer, token->value);
+    token_free(token);
   }
 
-  buffer_append(&name_buffer, first_token->value);
-
-  position_T* end_position = position_copy(first_token->location->end);
-  size_t range_end = first_token->range->to;
-
-  while (parser->current_token->type == TOKEN_CHARACTER && parser->current_token->value
-         && strcmp(parser->current_token->value, ".") == 0) {
-
-    token_T* dot_token = parser_advance(parser);
-
-    buffer_append(&name_buffer, dot_token->value);
-    position_free(end_position);
-
-    end_position = position_copy(dot_token->location->end);
-    range_end = dot_token->range->to;
-
-    token_free(dot_token);
+  parser_append_literal_node_from_buffer(parser, &buffer, children, start);
 
-    if (parser->current_token->type == TOKEN_IDENTIFIER) {
-      token_T* next_identifier = parser_advance(parser);
+  position_T* node_start = NULL;
+  position_T* node_end = NULL;
 
-      buffer_append(&name_buffer, next_identifier->value);
-      position_free(end_position);
+  if (children->size > 0) {
+    AST_NODE_T* first_child = array_get(children, 0);
+    AST_NODE_T* last_child = array_get(children, children->size - 1);
 
-      end_position = position_copy(next_identifier->location->end);
-      range_end = next_identifier->range->to;
-      token_free(next_identifier);
-    } else {
-      break;
-    }
+    node_start = position_copy(first_child->location->start);
+    node_end = position_copy(last_child->location->end);
+  } else {
+    node_start = position_copy(parser->current_token->location->start);
+    node_end = position_copy(parser->current_token->location->start);
   }
 
-  token_T* combined_token = calloc(1, sizeof(token_T));
-  combined_token->value = herb_strdup(name_buffer.value);
-  combined_token->type = TOKEN_IDENTIFIER;
-  combined_token->location =
-    location_from(start_position->line, start_position->column, end_position->line, end_position->column);
-
-  size_t range_start = at_token != NULL ? at_token->range->from : first_token->range->from;
-  combined_token->range = range_init(range_start, range_end);
-
   AST_HTML_ATTRIBUTE_NAME_NODE_T* attribute_name =
-    ast_html_attribute_name_node_init(combined_token, start_position, end_position, errors);
-
-  buffer_free(&name_buffer);
-  position_free(start_position);
-  position_free(end_position);
-  token_free(first_token);
-
-  if (at_token != NULL) { token_free(at_token); }
+    ast_html_attribute_name_node_init(children, node_start, node_end, errors);
 
-  token_free(combined_token);
+  position_free(start);
+  position_free(node_start);
+  position_free(node_end);
+  buffer_free(&buffer);
 
   return attribute_name;
 }
@@ -307,11 +374,89 @@ static AST_HTML_ATTRIBUTE_VALUE_NODE_T* parser_parse_quoted_html_attribute_value
       continue;
     }
 
+    if (token_is(parser, TOKEN_BACKSLASH)) {
+      lexer_state_snapshot_T saved_state = lexer_save_state(parser->lexer);
+
+      token_T* next_token = lexer_next_token(parser->lexer);
+
+      if (next_token && next_token->type == TOKEN_QUOTE && opening_quote != NULL
+          && strcmp(next_token->value, opening_quote->value) == 0) {
+        buffer_append(&buffer, parser->current_token->value);
+        buffer_append(&buffer, next_token->value);
+
+        token_free(parser->current_token);
+        token_free(next_token);
+
+        parser->current_token = lexer_next_token(parser->lexer);
+        continue;
+      } else {
+        lexer_restore_state(parser->lexer, saved_state);
+
+        if (next_token) { token_free(next_token); }
+      }
+    }
+
     buffer_append(&buffer, parser->current_token->value);
     token_free(parser->current_token);
+
     parser->current_token = lexer_next_token(parser->lexer);
   }
 
+  if (token_is(parser, TOKEN_QUOTE) && opening_quote != NULL
+      && strcmp(parser->current_token->value, opening_quote->value) == 0) {
+    lexer_state_snapshot_T saved_state = lexer_save_state(parser->lexer);
+
+    token_T* potential_closing = parser->current_token;
+    parser->current_token = lexer_next_token(parser->lexer);
+
+    if (token_is(parser, TOKEN_IDENTIFIER) || token_is(parser, TOKEN_CHARACTER)) {
+      append_unexpected_error(
+        "Unescaped quote character in attribute value",
+        "escaped quote (\\') or different quote style (\")",
+        opening_quote->value,
+        potential_closing->location->start,
+        potential_closing->location->end,
+        errors
+      );
+
+      lexer_restore_state(parser->lexer, saved_state);
+
+      token_free(parser->current_token);
+      parser->current_token = potential_closing;
+
+      buffer_append(&buffer, parser->current_token->value);
+      token_free(parser->current_token);
+      parser->current_token = lexer_next_token(parser->lexer);
+
+      while (!token_is(parser, TOKEN_EOF)
+             && !(
+               token_is(parser, TOKEN_QUOTE) && opening_quote != NULL
+               && strcmp(parser->current_token->value, opening_quote->value) == 0
+             )) {
+        if (token_is(parser, TOKEN_ERB_START)) {
+          parser_append_literal_node_from_buffer(parser, &buffer, children, start);
+
+          array_append(children, parser_parse_erb_tag(parser));
+
+          position_free(start);
+          start = position_copy(parser->current_token->location->start);
+
+          continue;
+        }
+
+        buffer_append(&buffer, parser->current_token->value);
+        token_free(parser->current_token);
+
+        parser->current_token = lexer_next_token(parser->lexer);
+      }
+    } else {
+      token_free(parser->current_token);
+      parser->current_token = potential_closing;
+
+      lexer_restore_state(parser->lexer, saved_state);
+    }
+  }
+
   parser_append_literal_node_from_buffer(parser, &buffer, children, start);
   position_free(start);
   buffer_free(&buffer);
@@ -439,18 +584,85 @@ static AST_HTML_ATTRIBUTE_VALUE_NODE_T* parser_parse_html_attribute_value(parser
 static AST_HTML_ATTRIBUTE_NODE_T* parser_parse_html_attribute(parser_T* parser) {
   AST_HTML_ATTRIBUTE_NAME_NODE_T* attribute_name = parser_parse_html_attribute_name(parser);
 
-  while (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
-    token_T* whitespace = parser_advance(parser);
-    token_free(whitespace);
+  if (parser->options && parser->options->track_whitespace) {
+    bool has_equals = (parser->current_token->type == TOKEN_EQUALS)
+                   || lexer_peek_for_token_type_after_whitespace(parser->lexer, TOKEN_EQUALS);
+
+    if (has_equals) {
+      buffer_T equals_buffer = buffer_new();
+      position_T* equals_start = NULL;
+      position_T* equals_end = NULL;
+      size_t range_start = 0;
+      size_t range_end = 0;
+
+      while (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
+        token_T* whitespace = parser_advance(parser);
+
+        if (equals_start == NULL) {
+          equals_start = position_copy(whitespace->location->start);
+          range_start = whitespace->range->from;
+        }
+
+        buffer_append(&equals_buffer, whitespace->value);
+        token_free(whitespace);
+      }
+
+      token_T* equals = parser_advance(parser);
+
+      if (equals_start == NULL) {
+        equals_start = position_copy(equals->location->start);
+        range_start = equals->range->from;
+      }
+
+      buffer_append(&equals_buffer, equals->value);
+      equals_end = position_copy(equals->location->end);
+      range_end = equals->range->to;
+      token_free(equals);
+
+      while (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
+        token_T* whitespace = parser_advance(parser);
+        buffer_append(&equals_buffer, whitespace->value);
+        equals_end = position_copy(whitespace->location->end);
+        range_end = whitespace->range->to;
+        token_free(whitespace);
+      }
+
+      token_T* equals_with_whitespace = calloc(1, sizeof(token_T));
+      equals_with_whitespace->type = TOKEN_EQUALS;
+      equals_with_whitespace->value = herb_strdup(equals_buffer.value);
+      equals_with_whitespace->location = location_init(equals_start, equals_end);
+      equals_with_whitespace->range = range_init(range_start, range_end);
+
+      buffer_free(&equals_buffer);
+
+      AST_HTML_ATTRIBUTE_VALUE_NODE_T* attribute_value = parser_parse_html_attribute_value(parser);
+
+      return ast_html_attribute_node_init(
+        attribute_name,
+        equals_with_whitespace,
+        attribute_value,
+        attribute_name->base.location->start,
+        attribute_value->base.location->end,
+        NULL
+      );
+    } else {
+      return ast_html_attribute_node_init(
+        attribute_name,
+        NULL,
+        NULL,
+        attribute_name->base.location->start,
+        attribute_name->base.location->end,
+        NULL
+      );
+    }
+  } else {
+    parser_consume_whitespace(parser, NULL);
   }
 
   token_T* equals = parser_consume_if_present(parser, TOKEN_EQUALS);
 
   if (equals != NULL) {
-    while (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
-      token_T* whitespace = parser_advance(parser);
-      token_free(whitespace);
-    }
+    parser_consume_whitespace(parser, NULL);
 
     AST_HTML_ATTRIBUTE_VALUE_NODE_T* attribute_value = parser_parse_html_attribute_value(parser);
 
@@ -478,30 +690,103 @@ static AST_HTML_ATTRIBUTE_NODE_T* parser_parse_html_attribute(parser_T* parser)
   );
 }
 
-static AST_HTML_OPEN_TAG_NODE_T* parser_parse_html_open_tag(parser_T* parser) {
-  array_T* errors = array_init(8);
-  array_T* children = array_init(8);
+static void parser_skip_erb_content(lexer_T* lexer) {
+  token_T* token = NULL;
 
-  token_T* tag_start = parser_consume_expected(parser, TOKEN_HTML_TAG_START, errors);
-  token_T* tag_name = parser_consume_expected(parser, TOKEN_IDENTIFIER, errors);
+  do {
+    token = lexer_next_token(lexer);
 
-  while (token_is_none_of(parser, TOKEN_HTML_TAG_END, TOKEN_HTML_TAG_SELF_CLOSE, TOKEN_EOF)) {
-    token_T* whitespace = parser_consume_if_present(parser, TOKEN_WHITESPACE);
+    if (token->type == TOKEN_ERB_END) {
+      token_free(token);
+      break;
+    }
 
-    if (whitespace != NULL) {
-      token_free(whitespace);
+    token_free(token);
+  } while (true);
+}
+
+static bool parser_lookahead_erb_is_attribute(lexer_T* lexer) {
+  token_T* after = NULL;
+
+  do {
+    after = lexer_next_token(lexer);
+
+    if (after->type == TOKEN_EQUALS) {
+      token_free(after);
+      return true;
+    }
+
+    if (after->type == TOKEN_WHITESPACE || after->type == TOKEN_NEWLINE) {
+      token_free(after);
       continue;
     }
 
-    token_T* newline = parser_consume_if_present(parser, TOKEN_NEWLINE);
+    if (after->type == TOKEN_IDENTIFIER || after->type == TOKEN_CHARACTER || after->type == TOKEN_DASH
+        || after->type == TOKEN_ERB_START) {
 
-    if (newline != NULL) {
-      token_free(newline);
+      if (after->type == TOKEN_ERB_START) {
+        token_free(after);
+        parser_skip_erb_content(lexer);
+      } else {
+        token_free(after);
+      }
       continue;
     }
 
-    if (parser->current_token->type == TOKEN_ERB_START) {
-      array_append(children, parser_parse_erb_tag(parser));
+    token_free(after);
+    return false;
+
+  } while (true);
+}
+
+static void parser_handle_erb_in_open_tag(parser_T* parser, array_T* children) {
+  bool is_output_tag = parser->current_token->value && strlen(parser->current_token->value) >= 3
+                    && strncmp(parser->current_token->value, "<%=", 3) == 0;
+
+  if (!is_output_tag) {
+    array_append(children, parser_parse_erb_tag(parser));
+
+    return;
+  }
+
+  lexer_T lexer_copy = *parser->lexer;
+
+  token_T* erb_start = lexer_next_token(&lexer_copy);
+  token_free(erb_start);
+  parser_skip_erb_content(&lexer_copy);
+
+  bool looks_like_attribute = parser_lookahead_erb_is_attribute(&lexer_copy);
+
+  if (looks_like_attribute) {
+    array_append(children, parser_parse_html_attribute(parser));
+  } else {
+    array_append(children, parser_parse_erb_tag(parser));
+  }
+}
+
+static void parser_handle_whitespace_in_open_tag(parser_T* parser, array_T* children) {
+  token_T* whitespace = parser_consume_if_present(parser, TOKEN_WHITESPACE);
+
+  if (whitespace != NULL) {
+    parser_handle_whitespace(parser, whitespace, children);
+    return;
+  }
+
+  token_T* newline = parser_consume_if_present(parser, TOKEN_NEWLINE);
+
+  if (newline != NULL) { parser_handle_whitespace(parser, newline, children); }
+}
+
+static AST_HTML_OPEN_TAG_NODE_T* parser_parse_html_open_tag(parser_T* parser) {
+  array_T* errors = array_init(8);
+  array_T* children = array_init(8);
+
+  token_T* tag_start = parser_consume_expected(parser, TOKEN_HTML_TAG_START, errors);
+  token_T* tag_name = parser_consume_expected(parser, TOKEN_IDENTIFIER, errors);
+
+  while (token_is_none_of(parser, TOKEN_HTML_TAG_END, TOKEN_HTML_TAG_SELF_CLOSE, TOKEN_EOF)) {
+    if (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
+      parser_handle_whitespace_in_open_tag(parser, children);
       continue;
     }
 
@@ -510,6 +795,11 @@ static AST_HTML_OPEN_TAG_NODE_T* parser_parse_html_open_tag(parser_T* parser) {
       continue;
     }
 
+    if (parser->current_token->type == TOKEN_ERB_START) {
+      parser_handle_erb_in_open_tag(parser, children);
+      continue;
+    }
+
     if (parser->current_token->type == TOKEN_AT) {
       array_append(children, parser_parse_html_attribute(parser));
       continue;
@@ -563,14 +853,15 @@ static AST_HTML_OPEN_TAG_NODE_T* parser_parse_html_open_tag(parser_T* parser) {
 
 static AST_HTML_CLOSE_TAG_NODE_T* parser_parse_html_close_tag(parser_T* parser) {
   array_T* errors = array_init(8);
+  array_T* children = array_init(8);
 
   token_T* tag_opening = parser_consume_expected(parser, TOKEN_HTML_TAG_START_CLOSE, errors);
+
+  parser_consume_whitespace(parser, children);
+
   token_T* tag_name = parser_consume_expected(parser, TOKEN_IDENTIFIER, errors);
 
-  while (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
-    token_T* whitespace = parser_advance(parser);
-    token_free(whitespace);
-  }
+  parser_consume_whitespace(parser, children);
 
   token_T* tag_closing = parser_consume_expected(parser, TOKEN_HTML_TAG_END, errors);
 
@@ -594,6 +885,7 @@ static AST_HTML_CLOSE_TAG_NODE_T* parser_parse_html_close_tag(parser_T* parser)
   AST_HTML_CLOSE_TAG_NODE_T* close_tag = ast_html_close_tag_node_init(
     tag_opening,
     tag_name,
+    children,
     tag_closing,
     tag_opening->location->start,
     tag_closing->location->end,
@@ -676,9 +968,7 @@ static AST_HTML_ELEMENT_NODE_T* parser_parse_html_element(parser_T* parser) {
   AST_HTML_OPEN_TAG_NODE_T* open_tag = parser_parse_html_open_tag(parser);
 
   // <tag />
-  if (open_tag->is_void || ast_node_is((AST_NODE_T*) open_tag, AST_HTML_SELF_CLOSE_TAG_NODE)) {
-    return parser_parse_html_self_closing_element(parser, open_tag);
-  }
+  if (open_tag->is_void) { return parser_parse_html_self_closing_element(parser, open_tag); }
 
   // <tag>, in void element list, and not in inside an <svg> element
   if (!open_tag->is_void && is_void_element(open_tag->tag_name->value) && !parser_in_svg_context(parser)) {
@@ -757,15 +1047,7 @@ static void parser_parse_foreign_content(parser_T* parser, array_T* children, ar
     }
 
     if (token_is(parser, TOKEN_HTML_TAG_START_CLOSE)) {
-      size_t saved_position = parser->lexer->current_position;
-      size_t saved_line = parser->lexer->current_line;
-      size_t saved_column = parser->lexer->current_column;
-      size_t saved_previous_position = parser->lexer->previous_position;
-      size_t saved_previous_line = parser->lexer->previous_line;
-      size_t saved_previous_column = parser->lexer->previous_column;
-
-      char saved_char = parser->lexer->current_character;
-      lexer_state_T saved_state = parser->lexer->state;
+      lexer_state_snapshot_T saved_state = lexer_save_state(parser->lexer);
 
       token_T* next_token = lexer_next_token(parser->lexer);
       bool is_potential_match = false;
@@ -774,14 +1056,7 @@ static void parser_parse_foreign_content(parser_T* parser, array_T* children, ar
         is_potential_match = parser_is_expected_closing_tag_name(next_token->value, parser->foreign_content_type);
       }
 
-      parser->lexer->current_position = saved_position;
-      parser->lexer->current_line = saved_line;
-      parser->lexer->current_column = saved_column;
-      parser->lexer->previous_position = saved_previous_position;
-      parser->lexer->previous_line = saved_previous_line;
-      parser->lexer->previous_column = saved_previous_column;
-      parser->lexer->current_character = saved_char;
-      parser->lexer->state = saved_state;
+      lexer_restore_state(parser->lexer, saved_state);
 
       if (next_token) { token_free(next_token); }
 
@@ -819,6 +1094,16 @@ static void parser_parse_in_data_state(parser_T* parser, array_T* children, arra
       continue;
     }
 
+    if (token_is(parser, TOKEN_XML_DECLARATION)) {
+      array_append(children, parser_parse_xml_declaration(parser));
+      continue;
+    }
+
+    if (token_is(parser, TOKEN_CDATA_START)) {
+      array_append(children, parser_parse_cdata(parser));
+      continue;
+    }
+
     if (token_is(parser, TOKEN_HTML_COMMENT_START)) {
       array_append(children, parser_parse_html_comment(parser));
       continue;
@@ -928,12 +1213,40 @@ AST_DOCUMENT_NODE_T* parser_parse(parser_T* parser) {
   return parser_parse_document(parser);
 }
 
+static void parser_handle_whitespace(parser_T* parser, token_T* whitespace_token, array_T* children) {
+  if (parser->options && parser->options->track_whitespace) {
+    array_T* errors = array_init(8);
+    AST_WHITESPACE_NODE_T* whitespace_node = ast_whitespace_node_init(
+      whitespace_token,
+      whitespace_token->location->start,
+      whitespace_token->location->end,
+      errors
+    );
+    array_append(children, whitespace_node);
+  }
+
+  token_free(whitespace_token);
+}
+
+static void parser_consume_whitespace(parser_T* parser, array_T* children) {
+  while (token_is_any_of(parser, TOKEN_WHITESPACE, TOKEN_NEWLINE)) {
+    token_T* whitespace = parser_advance(parser);
+
+    if (parser->options && parser->options->track_whitespace && children != NULL) {
+      parser_handle_whitespace(parser, whitespace, children);
+    } else {
+      token_free(whitespace);
+    }
+  }
+}
+
 void parser_free(parser_T* parser) {
   if (parser == NULL) { return; }
 
   if (parser->lexer != NULL) { lexer_free(parser->lexer); }
   if (parser->current_token != NULL) { token_free(parser->current_token); }
   if (parser->open_tags_stack != NULL) { array_free(&parser->open_tags_stack); }
+  if (parser->options != NULL) { free(parser->options); }
 
   free(parser);
 }