nokogumbo 2.0.0.pre.alpha → 2.0.4

data/gumbo-parser/src/string_buffer.c

@@ -79,7 +79,7 @@ void gumbo_string_buffer_append_codepoint (
 }
 
 void gumbo_string_buffer_append_string (
-  GumboStringPiece* str,
+  const GumboStringPiece* str,
   GumboStringBuffer* output
 ) {
   maybe_resize_string_buffer(str->length, output);

data/gumbo-parser/src/string_buffer.h

@@ -47,7 +47,7 @@ void gumbo_string_buffer_append_codepoint (
 
 // Appends a string onto the end of the GumboStringBuffer.
 void gumbo_string_buffer_append_string (
-  GumboStringPiece* str,
+  const GumboStringPiece* str,
   GumboStringBuffer* output
 );
 

data/gumbo-parser/src/token_buffer.c

@@ -0,0 +1,79 @@
+/*
+ Copyright 2018 Stephen Checkoway
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include <assert.h>
+
+#include "ascii.h"
+#include "token_buffer.h"
+#include "tokenizer.h"
+#include "util.h"
+
+struct GumboInternalCharacterToken {
+  GumboSourcePosition position;
+  GumboStringPiece original_text;
+  int c;
+};
+
+void gumbo_character_token_buffer_init(GumboCharacterTokenBuffer* buffer) {
+  buffer->data = NULL;
+  buffer->length = 0;
+  buffer->capacity = 0;
+}
+
+void gumbo_character_token_buffer_append (
+  const GumboToken* token,
+  GumboCharacterTokenBuffer* buffer
+) {
+  assert(token->type == GUMBO_TOKEN_WHITESPACE
+         || token->type == GUMBO_TOKEN_CHARACTER);
+  if (buffer->length == buffer->capacity) {
+    if (buffer->capacity == 0)
+      buffer->capacity = 10;
+    else
+      buffer->capacity *= 2;
+    size_t bytes = sizeof(*buffer->data) * buffer->capacity;
+    buffer->data = gumbo_realloc(buffer->data, bytes);
+  }
+  size_t index = buffer->length++;
+  buffer->data[index].position = token->position;
+  buffer->data[index].original_text = token->original_text;
+  buffer->data[index].c = token->v.character;
+}
+
+void gumbo_character_token_buffer_get (
+  const GumboCharacterTokenBuffer* buffer,
+  size_t index,
+  struct GumboInternalToken* output
+) {
+  assert(index < buffer->length);
+  int c = buffer->data[index].c;
+  output->type = gumbo_ascii_isspace(c)?
+    GUMBO_TOKEN_WHITESPACE : GUMBO_TOKEN_CHARACTER;
+  output->position = buffer->data[index].position;
+  output->original_text = buffer->data[index].original_text;
+  output->v.character = c;
+}
+
+void gumbo_character_token_buffer_clear(GumboCharacterTokenBuffer* buffer) {
+  buffer->length = 0;
+}
+
+void gumbo_character_token_buffer_destroy(GumboCharacterTokenBuffer* buffer) {
+  gumbo_free(buffer->data);
+  buffer->data = NULL;
+  buffer->length = 0;
+  buffer->capacity = 0;
+}

data/gumbo-parser/src/token_buffer.h

@@ -0,0 +1,71 @@
+/*
+ Copyright 2018 Stephen Checkoway
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#ifndef GUMBO_TOKEN_BUFFER_H
+#define GUMBO_TOKEN_BUFFER_H
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "gumbo.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct GumboInternalCharacterToken;
+struct GumboInternalToken;
+
+// A struct representing a growable sequence of character (and whitespace)
+// tokens.
+typedef struct {
+  // A pointer to the start of the sequence.
+  struct GumboInternalCharacterToken* data;
+
+  // The length of the sequence.
+  size_t length;
+
+  // The capacity of the buffer.
+  size_t capacity;
+} GumboCharacterTokenBuffer;
+
+// Initializes a new GumboCharacterTokenBuffer.
+void gumbo_character_token_buffer_init(GumboCharacterTokenBuffer* buffer);
+
+// Appends a character (or whitespace) token.
+void gumbo_character_token_buffer_append (
+  const struct GumboInternalToken* token,
+  GumboCharacterTokenBuffer* buffer
+);
+
+void gumbo_character_token_buffer_get (
+  const GumboCharacterTokenBuffer* buffer,
+  size_t index,
+  struct GumboInternalToken* output
+);
+
+// Reinitialize this string buffer. This clears it by setting length=0. It
+// does not zero out the buffer itself.
+void gumbo_character_token_buffer_clear(GumboCharacterTokenBuffer* buffer);
+
+// Deallocates this GumboCharacterTokenBuffer.
+void gumbo_character_token_buffer_destroy(GumboCharacterTokenBuffer* buffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // GUMBO_TOKEN_BUFFER_H

data/gumbo-parser/src/tokenizer.c

@@ -1,5 +1,7 @@
 /*
  Copyright 2010 Google Inc.
+ Copyright 2017-2018 Craig Barnes
+ Copyright 2018 Stephen Checkoway
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -18,10 +20,7 @@
  Coding conventions specific to this file:
 
  1. Functions that fill in a token should be named emit_*, and should be
-    followed immediately by a return from the tokenizer (true if no error
-    occurred, false if an error occurred). Sometimes the emit functions
-    themselves return a boolean so that they can be combined with the return
-    statement; in this case, they should match this convention.
+    followed immediately by a return from the tokenizer.
  2. Functions that shuffle data from temporaries to final API structures
     should be named finish_*, and be called just before the tokenizer exits the
     state that accumulates the temporary.
@@ -60,15 +59,18 @@
 #include "util.h"
 #include "vector.h"
 
-// Compared against _script_data_buffer to determine if we're in
+// Compared against _temporary_buffer to determine if we're in
 // double-escaped script mode.
 static const GumboStringPiece kScriptTag = {.data = "script", .length = 6};
 
-// An enum for the return value of each individual state.
+// An enum for the return value of each individual state. Each of the emit_*
+// functions should return EMIT_TOKEN and should be called as
+// return emit_foo(parser, ..., output);
+// Each of the handle_*_state functions that do not return emit_* should
+// instead return CONTINUE to indicate to gumbo_lex to continue lexing.
 typedef enum {
-  RETURN_ERROR,    // Return false (error) from the tokenizer.
-  RETURN_SUCCESS,  // Return true (success) from the tokenizer.
-  NEXT_CHAR        // Proceed to the next character and continue lexing.
+  EMIT_TOKEN,
+  CONTINUE,
 } StateResult;
 
 // This is a struct containing state necessary to build up a tag token,
@@ -103,12 +105,6 @@ typedef struct GumboInternalTagState {
   // the attribute value, but shouldn't overwrite the existing value.
   bool _drop_next_attr_value;
 
-  // The state that caused the tokenizer to switch into a character reference in
-  // attribute value state. This is used to set the additional allowed
-  // character, and is switched back to on completion. Initialized as the
-  // tokenizer enters the character reference state.
-  GumboTokenizerEnum _attr_value_state;
-
   // The last start tag to have been emitted by the tokenizer. This is
   // necessary to check for appropriate end tags.
   GumboTag _last_start_tag;
@@ -133,10 +129,10 @@ typedef struct GumboInternalTokenizerState {
   // "Reconsume the current input character in..."
   bool _reconsume_current_input;
 
-  // A flag indicating whether the current node is a foreign element. This is
-  // set by gumbo_tokenizer_set_is_current_node_foreign and checked in the
-  // markup declaration state.
-  bool _is_current_node_foreign;
+  // A flag indicating whether the adjusted current node is a foreign element.
+  // This is set by gumbo_tokenizer_set_is_adjusted_current_node_foreign and
+  // checked in the markup declaration state.
+  bool _is_adjusted_current_node_foreign;
 
   // A flag indicating whether the tokenizer is in a CDATA section. If so, then
   // text tokens emitted will be GUMBO_TOKEN_CDATA.
@@ -159,27 +155,24 @@ typedef struct GumboInternalTokenizerState {
 
   // A temporary buffer to accumulate characters, as described by the "temporary
   // buffer" phrase in the tokenizer spec. We use this in a somewhat unorthodox
-  // way: we record the specific character to go into the buffer, which may
-  // sometimes be a lowercased version of the actual input character. However,
-  // we *also* use utf8iterator_mark() to record the position at tag start.
-  // When we start flushing the temporary buffer, we set _temporary_buffer_emit
-  // to the start of it, and then increment it for each call to the tokenizer.
-  // We also call utf8iterator_reset(), and utf8iterator_next() through the
-  // input stream, so that tokens emitted by emit_char have the correct position
-  // and original text.
+  // way: In situations where the spec calls for inserting characters into the
+  // temporary buffer that exactly match the input in order to emit them as
+  // character tokens, we don't actually do it.
+  // Instead, we mark the input and reset the input to it using set_mark() and
+  // emit_from_mark(). We do use the temporary buffer for other uses such as
+  // DOCTYPEs, comments, and detecting escaped <script> tags.
   GumboStringBuffer _temporary_buffer;
 
-  // The current cursor position we're emitting from within
-  // _temporary_buffer.data. NULL whenever we're not flushing the buffer.
-  const char* _temporary_buffer_emit;
+  // The position to resume normal operation after we start emitting from the
+  // mark. NULL whenever we're not emitting from the mark.
+  const char* _resume_pos;
+
+  // The character reference state uses a return state to return to the state
+  // it was invoked from.
+  GumboTokenizerEnum _return_state;
 
-  // The temporary buffer is also used by the spec to check whether we should
-  // enter the script data double escaped state, but we can't use the same
-  // buffer for both because we have to flush out "<s" as emits while still
-  // maintaining the context that will eventually become "script". This is a
-  // separate buffer that's used in place of the temporary buffer for states
-  // that may enter the script data double escape start state.
-  GumboStringBuffer _script_data_buffer;
+  // Numeric character reference.
+  uint32_t _character_reference_code;
 
   // Pointer to the beginning of the current token in the original buffer; used
   // to record the original text.
@@ -201,123 +194,66 @@ typedef struct GumboInternalTokenizerState {
   Utf8Iterator _input;
 } GumboTokenizerState;
 
-// Adds an ERR_UNEXPECTED_CODE_POINT parse error to the parser's error struct.
+// Adds a parse error to the parser's error struct.
 static void tokenizer_add_parse_error (
   GumboParser* parser,
   GumboErrorType type
 ) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   GumboError* error = gumbo_add_error(parser);
   if (!error) {
     return;
   }
+  const Utf8Iterator* input = &tokenizer->_input;
+  utf8iterator_get_position(input, &error->position);
+  error->original_text.data = utf8iterator_get_char_pointer(input);
+  error->original_text.length = utf8iterator_get_width(input);
+  error->type = type;
+  error->v.tokenizer.state = tokenizer->_state;
+  error->v.tokenizer.codepoint = utf8iterator_current(input);
+}
+
+// Adds an error pointing at the start of the character reference.
+static void tokenizer_add_char_ref_error (
+  struct GumboInternalParser* parser,
+  GumboErrorType type,
+  int codepoint
+) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
-  utf8iterator_get_position(&tokenizer->_input, &error->position);
-  error->original_text = utf8iterator_get_char_pointer(&tokenizer->_input);
+  GumboError* error = gumbo_add_error(parser);
+  if (!error)
+    return;
+  Utf8Iterator* input = &tokenizer->_input;
   error->type = type;
-  error->v.tokenizer.codepoint = utf8iterator_current(&tokenizer->_input);
-  switch (tokenizer->_state) {
-    case GUMBO_LEX_DATA:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_DATA;
-      break;
-    case GUMBO_LEX_CHAR_REF_IN_DATA:
-    case GUMBO_LEX_CHAR_REF_IN_RCDATA:
-    case GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_CHAR_REF;
-      break;
-    case GUMBO_LEX_RCDATA:
-    case GUMBO_LEX_RCDATA_LT:
-    case GUMBO_LEX_RCDATA_END_TAG_OPEN:
-    case GUMBO_LEX_RCDATA_END_TAG_NAME:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_RCDATA;
-      break;
-    case GUMBO_LEX_RAWTEXT:
-    case GUMBO_LEX_RAWTEXT_LT:
-    case GUMBO_LEX_RAWTEXT_END_TAG_OPEN:
-    case GUMBO_LEX_RAWTEXT_END_TAG_NAME:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_RAWTEXT;
-      break;
-    case GUMBO_LEX_PLAINTEXT:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_PLAINTEXT;
-      break;
-    case GUMBO_LEX_SCRIPT:
-    case GUMBO_LEX_SCRIPT_LT:
-    case GUMBO_LEX_SCRIPT_END_TAG_OPEN:
-    case GUMBO_LEX_SCRIPT_END_TAG_NAME:
-    case GUMBO_LEX_SCRIPT_ESCAPED_START:
-    case GUMBO_LEX_SCRIPT_ESCAPED_START_DASH:
-    case GUMBO_LEX_SCRIPT_ESCAPED:
-    case GUMBO_LEX_SCRIPT_ESCAPED_DASH:
-    case GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH:
-    case GUMBO_LEX_SCRIPT_ESCAPED_LT:
-    case GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN:
-    case GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME:
-    case GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START:
-    case GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED:
-    case GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH:
-    case GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH:
-    case GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT:
-    case GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_SCRIPT;
-      break;
-    case GUMBO_LEX_TAG_OPEN:
-    case GUMBO_LEX_END_TAG_OPEN:
-    case GUMBO_LEX_TAG_NAME:
-    case GUMBO_LEX_BEFORE_ATTR_NAME:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_TAG;
-      break;
-    case GUMBO_LEX_SELF_CLOSING_START_TAG:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_SELF_CLOSING_TAG;
-      break;
-    case GUMBO_LEX_ATTR_NAME:
-    case GUMBO_LEX_AFTER_ATTR_NAME:
-    case GUMBO_LEX_BEFORE_ATTR_VALUE:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_ATTR_NAME;
-      break;
-    case GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED:
-    case GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED:
-    case GUMBO_LEX_ATTR_VALUE_UNQUOTED:
-    case GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_ATTR_VALUE;
-      break;
-    case GUMBO_LEX_BOGUS_COMMENT:
-    case GUMBO_LEX_COMMENT_START:
-    case GUMBO_LEX_COMMENT_START_DASH:
-    case GUMBO_LEX_COMMENT:
-    case GUMBO_LEX_COMMENT_END_DASH:
-    case GUMBO_LEX_COMMENT_END:
-    case GUMBO_LEX_COMMENT_END_BANG:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_COMMENT;
-      break;
-    case GUMBO_LEX_MARKUP_DECLARATION:
-    case GUMBO_LEX_DOCTYPE:
-    case GUMBO_LEX_BEFORE_DOCTYPE_NAME:
-    case GUMBO_LEX_DOCTYPE_NAME:
-    case GUMBO_LEX_AFTER_DOCTYPE_NAME:
-    case GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD:
-    case GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID:
-    case GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED:
-    case GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED:
-    case GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID:
-    case GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID:
-    case GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD:
-    case GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID:
-    case GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED:
-    case GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED:
-    case GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID:
-    case GUMBO_LEX_BOGUS_DOCTYPE:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_DOCTYPE;
-      break;
-    case GUMBO_LEX_CDATA:
-      error->v.tokenizer.state = GUMBO_ERR_TOKENIZER_CDATA;
-      break;
-  }
+  error->position = utf8iterator_get_mark_position(input);
+  const char* mark = utf8iterator_get_mark_pointer(input);
+  error->original_text.data = mark;
+  error->original_text.length = utf8iterator_get_char_pointer(input) - mark;
+  error->v.tokenizer.state = tokenizer->_state;
+  error->v.tokenizer.codepoint = codepoint;
+}
+
+// Adds an error pointing at the start of the token.
+static void tokenizer_add_token_parse_error (
+  GumboParser* parser,
+  GumboErrorType type
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboError* error = gumbo_add_error(parser);
+  if (!error)
+    return;
+  Utf8Iterator* input = &tokenizer->_input;
+  error->type = type;
+  error->position = tokenizer->_token_start_pos;
+  error->original_text.data = tokenizer->_token_start;
+  error->original_text.length =
+    utf8iterator_get_char_pointer(input) - tokenizer->_token_start;
+  error->v.tokenizer.state = tokenizer->_state;
+  error->v.tokenizer.codepoint = 0;
 }
 
 static bool is_alpha(int c) {
-  // We don't use the ISO C isalpha() function here because it depends
-  // on the current locale, whereas the behavior in the HTML5 spec is
-  // locale-independent.
-  return ((unsigned) c | 32) - 'a' < 26;
+  return gumbo_ascii_isalpha(c);
 }
 
 static int ensure_lowercase(int c) {
@@ -347,24 +283,9 @@ static GumboTokenType get_char_token_type(bool is_in_cdata, int c) {
 }
 
 // Starts recording characters in the temporary buffer.
-// Because this needs to reset the utf8iterator_mark to the beginning of the
-// text that will eventually be emitted, it needs to be called a couple of
-// states before the spec says "Set the temporary buffer to the empty string".
-// In general, this should be called whenever there's a transition to a
-// "less-than sign state". The initial < and possibly / then need to be
-// appended to the temporary buffer, their presence needs to be accounted for in
-// states that compare the temporary buffer against a literal value, and
-// spec stanzas that say "emit a < and / character token along with a character
-// token for each character in the temporary buffer" need to be adjusted to
-// account for the presence of the < and / inside the temporary buffer.
 static void clear_temporary_buffer(GumboParser* parser) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
-  assert(!tokenizer->_temporary_buffer_emit);
-  utf8iterator_mark(&tokenizer->_input);
   gumbo_string_buffer_clear(&tokenizer->_temporary_buffer);
-  // The temporary buffer and script data buffer are the same object in the
-  // spec, so the script data buffer should be cleared as well.
-  gumbo_string_buffer_clear(&tokenizer->_script_data_buffer);
 }
 
 // Appends a codepoint to the temporary buffer.
@@ -378,25 +299,20 @@ static void append_char_to_temporary_buffer (
   );
 }
 
-#ifndef NDEBUG
-static bool temporary_buffer_equals__ (
-  const GumboParser* parser,
-  const char* text,
-  size_t text_len
+static void append_string_to_temporary_buffer (
+  GumboParser* parser,
+  const GumboStringPiece* str
 ) {
-  const GumboStringBuffer* buf = &parser->_tokenizer_state->_temporary_buffer;
-  return
-    text_len == buf->length
-    && memcmp(buf->data, text, text_len) == 0;
+  gumbo_string_buffer_append_string (
+    str,
+    &parser->_tokenizer_state->_temporary_buffer
+  );
 }
 
-#define temporary_buffer_equals(parser, text) \
-  temporary_buffer_equals__(parser, "" text, sizeof(text) - 1)
 
 static bool temporary_buffer_is_empty(const GumboParser* parser) {
   return parser->_tokenizer_state->_temporary_buffer.length == 0;
 }
-#endif
 
 static void doc_type_state_init(GumboParser* parser) {
   GumboTokenDocType* doc_type_state =
@@ -493,56 +409,49 @@ static void finish_doctype_system_id(GumboParser* parser) {
 }
 
 // Writes a single specified character to the output token.
-static void emit_char(GumboParser* parser, int c, GumboToken* output) {
+static StateResult emit_char(GumboParser* parser, int c, GumboToken* output) {
   output->type = get_char_token_type(parser->_tokenizer_state->_is_in_cdata, c);
   output->v.character = c;
   finish_token(parser, output);
+  return EMIT_TOKEN;
 }
 
 // Writes a replacement character token and records a parse error.
-// Always returns RETURN_ERROR, per gumbo_lex return value.
+// Always returns EMIT_TOKEN, per gumbo_lex return value.
 static StateResult emit_replacement_char(
     GumboParser* parser, GumboToken* output) {
   // In all cases, this is because of a null byte in the input stream.
-  tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+  tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
   emit_char(parser, kUtf8ReplacementChar, output);
-  return RETURN_ERROR;
+  return EMIT_TOKEN;
 }
 
-// Writes an EOF character token. Always returns RETURN_SUCCESS.
+// Writes an EOF character token. Always returns EMIT_TOKEN.
 static StateResult emit_eof(GumboParser* parser, GumboToken* output) {
-  emit_char(parser, -1, output);
-  return RETURN_SUCCESS;
-}
-
-// Writes the current input character out as a character token.
-// Always returns RETURN_SUCCESS.
-static bool emit_current_char(GumboParser* parser, GumboToken* output) {
-  emit_char(
-      parser, utf8iterator_current(&parser->_tokenizer_state->_input), output);
-  return RETURN_SUCCESS;
+  return emit_char(parser, -1, output);
 }
 
 // Writes out a doctype token, copying it from the tokenizer state.
-static void emit_doctype(GumboParser* parser, GumboToken* output) {
+static StateResult emit_doctype(GumboParser* parser, GumboToken* output) {
   output->type = GUMBO_TOKEN_DOCTYPE;
   output->v.doc_type = parser->_tokenizer_state->_doc_type_state;
   finish_token(parser, output);
   doc_type_state_init(parser);
+  return EMIT_TOKEN;
 }
 
 // Debug-only function that explicitly sets the attribute vector data to NULL so
 // it can be asserted on tag creation, verifying that there are no memory leaks.
 static void mark_tag_state_as_empty(GumboTagState* tag_state) {
   UNUSED_IF_NDEBUG(tag_state);
-#ifndef NDEBUG
   tag_state->_name = NULL;
+#ifndef NDEBUG
   tag_state->_attributes = kGumboEmptyVector;
 #endif
 }
 
 // Writes out the current tag as a start or end tag token.
-// Always returns RETURN_SUCCESS.
+// Always returns EMIT_TOKEN.
 static StateResult emit_current_tag(GumboParser* parser, GumboToken* output) {
   GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
   if (tag_state->_is_start_tag) {
@@ -559,7 +468,10 @@ static StateResult emit_current_tag(GumboParser* parser, GumboToken* output) {
     output->type = GUMBO_TOKEN_END_TAG;
     output->v.end_tag.tag = tag_state->_tag;
     output->v.end_tag.name = tag_state->_name;
-    output->v.end_tag.is_self_closing = tag_state->_is_self_closing;
+    if (tag_state->_is_self_closing)
+      tokenizer_add_token_parse_error(parser, GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS);
+    if (tag_state->_attributes.length > 0)
+      tokenizer_add_token_parse_error(parser, GUMBO_ERR_END_TAG_WITH_ATTRIBUTES);
     // In end tags, ownership of the attributes vector is not transferred to the
     // token, but it's still initialized as normal, so it must be manually
     // deallocated. There may also be attributes to destroy, in certain broken
@@ -582,7 +494,7 @@ static StateResult emit_current_tag(GumboParser* parser, GumboToken* output) {
   assert(output->original_text.length >= 2);
   assert(output->original_text.data[0] == '<');
   assert(output->original_text.data[output->original_text.length - 1] == '>');
-  return RETURN_SUCCESS;
+  return EMIT_TOKEN;
 }
 
 // In some states, we speculatively start a tag, but don't know whether it'll be
@@ -600,90 +512,59 @@ static void abandon_current_tag(GumboParser* parser) {
   gumbo_debug("Abandoning current tag.\n");
 }
 
-// Wraps the gumbo_consume_char_ref function to handle its output and make the
-// appropriate TokenizerState modifications. Returns RETURN_ERROR if a parse
-// error occurred, RETURN_SUCCESS otherwise.
-static StateResult emit_char_ref (
-  GumboParser* parser,
-  int additional_allowed_char,
-  bool UNUSED_ARG(is_in_attribute),
-  GumboToken* output
-) {
-  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
-  OneOrTwoCodepoints char_ref;
-  bool status = gumbo_consume_char_ref (
-    parser,
-    &tokenizer->_input,
-    additional_allowed_char,
-    false,
-    &char_ref
-  );
-  if (char_ref.first != kGumboNoChar) {
-    // gumbo_consume_char_ref ends with the iterator pointing at the next
-    // character, so we need to be sure not advance it again before
-    // reading the next token.
-    tokenizer->_reconsume_current_input = true;
-    emit_char(parser, char_ref.first, output);
-    tokenizer->_buffered_emit_char = char_ref.second;
-  } else {
-    emit_char(parser, '&', output);
-  }
-  return status ? RETURN_SUCCESS : RETURN_ERROR;
-}
-
 // Emits a comment token. Comments use the temporary buffer to accumulate their
 // data, and then it's copied over and released to the 'text' field of the
-// GumboToken union. Always returns RETURN_SUCCESS.
+// GumboToken union. Always returns EMIT_TOKEN.
 static StateResult emit_comment(GumboParser* parser, GumboToken* output) {
   output->type = GUMBO_TOKEN_COMMENT;
   finish_temporary_buffer(parser, &output->v.text);
   finish_token(parser, output);
-  return RETURN_SUCCESS;
+  return EMIT_TOKEN;
 }
 
-// Checks to see we should be flushing accumulated characters in the temporary
-// buffer, and fills the output token with the next output character if so.
-// Returns true if a character has been emitted and the tokenizer should
-// immediately return, false if we're at the end of the temporary buffer and
-// should resume normal operation.
-static bool maybe_emit_from_temporary_buffer(
-    GumboParser* parser, GumboToken* output) {
+static void set_mark(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  utf8iterator_mark(&tokenizer->_input);
+}
+
+// Checks to see we should be emitting characters from the mark, and fills the
+// output token with the next output character if so.
+// Returns EMIT_TOKEN if a character has been emitted and the tokenizer should
+// immediately return, CONTINUE if we should resume normal operation.
+static StateResult maybe_emit_from_mark (
+    GumboParser* parser,
+    GumboToken* output
+) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
-  const char* c = tokenizer->_temporary_buffer_emit;
-  GumboStringBuffer* buffer = &tokenizer->_temporary_buffer;
+  const char* pos = tokenizer->_resume_pos;
 
-  if (!c || c >= buffer->data + buffer->length) {
-    tokenizer->_temporary_buffer_emit = NULL;
-    return false;
+  if (!pos)
+    return CONTINUE;
+  if (utf8iterator_get_char_pointer(&tokenizer->_input) >= pos) {
+    tokenizer->_resume_pos = NULL;
+    return CONTINUE;
   }
 
-  assert(*c == utf8iterator_current(&tokenizer->_input));
-  // emit_char also advances the input stream. We need to do some juggling of
-  // the _reconsume_current_input flag to get the proper behavior when emitting
-  // previous tokens. Basically, _reconsume_current_input should *never* be set
-  // when emitting anything from the temporary buffer, since those characters
-  // have already been advanced past. However, it should be preserved so that
-  // when the *next* character is encountered again, the tokenizer knows not to
-  // advance past it.
-  bool saved_reconsume_state = tokenizer->_reconsume_current_input;
-  tokenizer->_reconsume_current_input = false;
-  emit_char(parser, *c, output);
-  ++tokenizer->_temporary_buffer_emit;
-  tokenizer->_reconsume_current_input = saved_reconsume_state;
-  return true;
-}
-
-// Sets up the tokenizer to begin flushing the temporary buffer.
-// This resets the input iterator stream to the start of the last tag, sets up
-// _temporary_buffer_emit, and then (if the temporary buffer is non-empty) emits
-// the first character in it. It returns true if a character was emitted, false
-// otherwise.
-static bool emit_temporary_buffer(GumboParser* parser, GumboToken* output) {
+  // emit_char advances the input stream. _reconsume_current_input should
+  // *never* be set when emitting from the mark since those characters have
+  // already been advanced past.
+  assert(!tokenizer->_reconsume_current_input);
+  return emit_char(parser, utf8iterator_current(&tokenizer->_input), output);
+}
+
+// Sets up the tokenizer to begin emitting from the mark up to, but not
+// including, the current code point. This resets the input iterator stream to
+// the mark, sets up _resume_pos, and then emits the first character in it.
+// Returns EMIT_TOKEN.
+static StateResult emit_from_mark(GumboParser* parser, GumboToken* output) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
-  assert(tokenizer->_temporary_buffer.data);
+  tokenizer->_resume_pos = utf8iterator_get_char_pointer(&tokenizer->_input);
   utf8iterator_reset(&tokenizer->_input);
-  tokenizer->_temporary_buffer_emit = tokenizer->_temporary_buffer.data;
-  return maybe_emit_from_temporary_buffer(parser, output);
+  // Now that we have reset the input, we need to advance through it.
+  tokenizer->_reconsume_current_input = false;
+  StateResult result = maybe_emit_from_mark(parser, output);
+  assert(result == EMIT_TOKEN);
+  return result;
 }
 
 // Appends a codepoint to the current tag buffer. If
@@ -703,6 +584,19 @@ static void append_char_to_tag_buffer (
   gumbo_string_buffer_append_codepoint(codepoint, buffer);
 }
 
+// Like above but append a string.
+static void append_string_to_tag_buffer (
+  GumboParser* parser,
+  GumboStringPiece* str,
+  bool reinitilize_position_on_first
+) {
+  GumboStringBuffer* buffer = &parser->_tokenizer_state->_tag_state._buffer;
+  if (buffer->length == 0 && reinitilize_position_on_first) {
+    reset_tag_buffer_start_point(parser);
+  }
+  gumbo_string_buffer_append_string(str, buffer);
+}
+
 // (Re-)initialize the tag buffer. This also resets the original_text pointer
 // and _start_pos field to point to the current position.
 static void initialize_tag_buffer(GumboParser* parser) {
@@ -713,6 +607,70 @@ static void initialize_tag_buffer(GumboParser* parser) {
   reset_tag_buffer_start_point(parser);
 }
 
+// https://html.spec.whatwg.org/multipage/parsing.html#charref-in-attribute
+static bool character_reference_part_of_attribute(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  switch (tokenizer->_return_state) {
+  case GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED:
+  case GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED:
+  case GUMBO_LEX_ATTR_VALUE_UNQUOTED:
+    return true;
+  default:
+    return false;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#flush-code-points-consumed-as-a-character-reference
+// For each code point in the temporary buffer, add to the current attribute
+// value if the character reference was consumed as part of an attribute or
+// emit the code point as a character token.
+static StateResult flush_code_points_consumed_as_character_reference (
+  GumboParser* parser,
+  GumboToken* output
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  if (character_reference_part_of_attribute(parser)) {
+    const char *start = utf8iterator_get_mark_pointer(&tokenizer->_input);
+    assert(start);
+    GumboStringPiece str = {
+      .data = start,
+      .length = utf8iterator_get_char_pointer(&tokenizer->_input) - start,
+    };
+    bool unquoted = tokenizer->_return_state == GUMBO_LEX_ATTR_VALUE_UNQUOTED;
+    append_string_to_tag_buffer(parser, &str, unquoted);
+    return CONTINUE;
+  }
+  return emit_from_mark(parser, output);
+}
+
+// After a character reference has been successfully constructed, the standard
+// says to set the temporary buffer equal to the empty string, append the code
+// point(s) associated with the reference and flush code points consumed as a
+// character reference.
+// https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
+// That doesn't work for us because we use the temporary buffer in lock step
+// with the input for position and that would fail if we inserted a different
+// number of code points. So duplicate a bit of the above logic.
+static StateResult flush_char_ref (
+  GumboParser* parser,
+  int first,
+  int second,
+  GumboToken* output
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  if (character_reference_part_of_attribute(parser)) {
+    bool unquoted = tokenizer->_return_state == GUMBO_LEX_ATTR_VALUE_UNQUOTED;
+    append_char_to_tag_buffer(parser, first, unquoted);
+    if (second != kGumboNoChar)
+      append_char_to_tag_buffer(parser, second, unquoted);
+    return CONTINUE;
+  }
+  tokenizer->_buffered_emit_char = second;
+  return emit_char(parser, first, output);
+}
+
+
 // Initializes the tag_state to start a new tag, keeping track of the opening
 // positions and original text. Takes a boolean indicating whether this is a
 // start or end tag.
@@ -725,7 +683,6 @@ static void start_new_tag(GumboParser* parser, bool is_start_tag) {
   assert(is_alpha(c));
 
   initialize_tag_buffer(parser);
-  gumbo_string_buffer_append_codepoint(c, &tag_state->_buffer);
 
   assert(tag_state->_name == NULL);
   assert(tag_state->_attributes.data == NULL);
@@ -765,7 +722,10 @@ static void copy_over_original_tag_text (
   original_text->data = tag_state->_original_text;
   original_text->length = utf8iterator_get_char_pointer(&tokenizer->_input) -
                           tag_state->_original_text;
-  if (original_text->data[original_text->length - 1] == '\r') {
+  if (
+    original_text->length
+    && original_text->data[original_text->length - 1] == '\r'
+  ) {
     // Since \r is skipped by the UTF-8 iterator, it can sometimes end up
     // appended to the end of original text even when it's really the first part
     // of the next character. If we detect this situation, shrink the length of
@@ -801,40 +761,45 @@ static void finish_tag_name(GumboParser* parser) {
 }
 
 // Adds an ERR_DUPLICATE_ATTR parse error to the parser's error struct.
-static void add_duplicate_attr_error (
-  GumboParser* parser,
-  int original_index,
-  int new_index
-) {
+static void add_duplicate_attr_error(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   GumboError* error = gumbo_add_error(parser);
   if (!error) {
     return;
   }
   GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
-  error->type = GUMBO_ERR_DUPLICATE_ATTR;
+  error->type = GUMBO_ERR_DUPLICATE_ATTRIBUTE;
   error->position = tag_state->_start_pos;
-  error->original_text = tag_state->_original_text;
-  error->v.duplicate_attr.original_index = original_index;
-  error->v.duplicate_attr.new_index = new_index;
-  copy_over_tag_buffer(parser, &error->v.duplicate_attr.name);
-  reinitialize_tag_buffer(parser);
+  error->original_text.data = tag_state->_original_text;
+  error->original_text.length =
+    utf8iterator_get_char_pointer(&tokenizer->_input) - error->original_text.data;
+  error->v.tokenizer.state = tokenizer->_state;
 }
 
 // Creates a new attribute in the current tag, copying the current tag buffer to
 // the attribute's name. The attribute's value starts out as the empty string
 // (following the "Boolean attributes" section of the spec) and is only
 // overwritten on finish_attribute_value(). If the attribute has already been
-// specified, the new attribute is dropped, a parse error is added, and the
-// function returns false. Otherwise, this returns true.
-static bool finish_attribute_name(GumboParser* parser) {
+// specified, the new attribute is dropped and a parse error is added
+static void finish_attribute_name(GumboParser* parser) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   GumboTagState* tag_state = &tokenizer->_tag_state;
+  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
+
+  int max_attributes = parser->_options->max_attributes;
+  if (unlikely(max_attributes >= 0 && attributes->length >= (unsigned int) max_attributes)) {
+    parser->_output->status = GUMBO_STATUS_TOO_MANY_ATTRIBUTES;
+    gumbo_debug("Attributes limit exceeded.\n");
+    reinitialize_tag_buffer(parser);
+    tag_state->_drop_next_attr_value = true;
+    return;
+  }
+
   // May've been set by a previous attribute without a value; reset it here.
   tag_state->_drop_next_attr_value = false;
   assert(tag_state->_attributes.data);
   assert(tag_state->_attributes.capacity);
 
-  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
   for (unsigned int i = 0; i < attributes->length; ++i) {
     GumboAttribute* attr = attributes->data[i];
     if (
@@ -846,9 +811,10 @@ static bool finish_attribute_name(GumboParser* parser) {
       )
     ) {
       // Identical attribute; bail.
-      add_duplicate_attr_error(parser, i, attributes->length);
+      add_duplicate_attr_error(parser);
+      reinitialize_tag_buffer(parser);
       tag_state->_drop_next_attr_value = true;
-      return false;
+      return;
     }
   }
 
@@ -870,7 +836,6 @@ static bool finish_attribute_name(GumboParser* parser) {
   );
   gumbo_vector_add(attr, attributes);
   reinitialize_tag_buffer(parser);
-  return true;
 }
 
 // Finishes an attribute value. This sets the value of the most recently added
@@ -911,22 +876,23 @@ void gumbo_tokenizer_state_init (
   GumboTokenizerState* tokenizer = gumbo_alloc(sizeof(GumboTokenizerState));
   parser->_tokenizer_state = tokenizer;
   gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+  tokenizer->_return_state = GUMBO_LEX_DATA;
+  tokenizer->_character_reference_code = 0;
   tokenizer->_reconsume_current_input = false;
-  tokenizer->_is_current_node_foreign = false;
+  tokenizer->_is_adjusted_current_node_foreign = false;
   tokenizer->_is_in_cdata = false;
   tokenizer->_tag_state._last_start_tag = GUMBO_TAG_LAST;
   tokenizer->_tag_state._name = NULL;
 
   tokenizer->_buffered_emit_char = kGumboNoChar;
   gumbo_string_buffer_init(&tokenizer->_temporary_buffer);
-  tokenizer->_temporary_buffer_emit = NULL;
+  tokenizer->_resume_pos = NULL;
 
   mark_tag_state_as_empty(&tokenizer->_tag_state);
 
-  gumbo_string_buffer_init(&tokenizer->_script_data_buffer);
-  tokenizer->_token_start = text;
   utf8iterator_init(parser, text, text_length, &tokenizer->_input);
   utf8iterator_get_position(&tokenizer->_input, &tokenizer->_token_start_pos);
+  tokenizer->_token_start = utf8iterator_get_char_pointer(&tokenizer->_input);
   doc_type_state_init(parser);
 }
 
@@ -936,7 +902,6 @@ void gumbo_tokenizer_state_destroy(GumboParser* parser) {
   assert(tokenizer->_doc_type_state.public_identifier == NULL);
   assert(tokenizer->_doc_type_state.system_identifier == NULL);
   gumbo_string_buffer_destroy(&tokenizer->_temporary_buffer);
-  gumbo_string_buffer_destroy(&tokenizer->_script_data_buffer);
   assert(tokenizer->_tag_state._name == NULL);
   assert(tokenizer->_tag_state._attributes.data == NULL);
   gumbo_free(tokenizer);
@@ -946,17 +911,23 @@ void gumbo_tokenizer_set_state(GumboParser* parser, GumboTokenizerEnum state) {
   parser->_tokenizer_state->_state = state;
 }
 
-void gumbo_tokenizer_set_is_current_node_foreign (
+void gumbo_tokenizer_set_is_adjusted_current_node_foreign (
   GumboParser* parser,
   bool is_foreign
 ) {
-  if (is_foreign != parser->_tokenizer_state->_is_current_node_foreign) {
+  if (is_foreign != parser->_tokenizer_state->_is_adjusted_current_node_foreign) {
     gumbo_debug (
       "Toggling is_current_node_foreign to %s.\n",
       is_foreign ? "true" : "false"
     );
   }
-  parser->_tokenizer_state->_is_current_node_foreign = is_foreign;
+  parser->_tokenizer_state->_is_adjusted_current_node_foreign = is_foreign;
+}
+
+static void reconsume_in_state(GumboParser* parser, GumboTokenizerEnum state) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  tokenizer->_reconsume_current_input = true;
+  tokenizer->_state = state;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#data-state
@@ -968,37 +939,24 @@ static StateResult handle_data_state (
 ) {
   switch (c) {
     case '&':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHAR_REF_IN_DATA);
-      // The char_ref machinery expects to be on the & so it can mark that
-      // and return to it if the text isn't a char ref, so we need to
-      // reconsume it.
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_DATA;
+      return CONTINUE;
     case '<':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_TAG_OPEN);
-      clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, '<');
-      return NEXT_CHAR;
+      set_mark(parser);
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      emit_char(parser, c, output);
-      return RETURN_ERROR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      return emit_char(parser, c, output);
+    case -1:
+      return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
-// https://html.spec.whatwg.org/multipage/parsing.html#character-reference-in-data-state
-static StateResult handle_char_ref_in_data_state (
-  GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
-  int UNUSED_ARG(c),
-  GumboToken* output
-) {
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-  return emit_char_ref(parser, ' ', false, output);
-}
-
 // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-state
 static StateResult handle_rcdata_state (
   GumboParser* parser,
@@ -1008,34 +966,23 @@ static StateResult handle_rcdata_state (
 ) {
   switch (c) {
     case '&':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHAR_REF_IN_RCDATA);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_RCDATA;
+      return CONTINUE;
     case '<':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA_LT);
-      clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, '<');
-      return NEXT_CHAR;
+      set_mark(parser);
+      return CONTINUE;
     case '\0':
       return emit_replacement_char(parser, output);
     case -1:
       return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
-// https://html.spec.whatwg.org/multipage/parsing.html#character-reference-in-rcdata-state
-static StateResult handle_char_ref_in_rcdata_state (
-  GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
-  int UNUSED_ARG(c),
-  GumboToken* output
-) {
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA);
-  return emit_char_ref(parser, ' ', false, output);
-}
-
 // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-state
 static StateResult handle_rawtext_state (
   GumboParser* parser,
@@ -1046,20 +993,19 @@ static StateResult handle_rawtext_state (
   switch (c) {
     case '<':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT_LT);
-      clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, '<');
-      return NEXT_CHAR;
+      set_mark(parser);
+      return CONTINUE;
     case '\0':
       return emit_replacement_char(parser, output);
     case -1:
       return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-state
-static StateResult handle_script_state (
+static StateResult handle_script_data_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1067,16 +1013,15 @@ static StateResult handle_script_state (
 ) {
   switch (c) {
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_LT);
-      clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, '<');
-      return NEXT_CHAR;
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_LT);
+      set_mark(parser);
+      return CONTINUE;
     case '\0':
       return emit_replacement_char(parser, output);
     case -1:
       return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
@@ -1093,75 +1038,75 @@ static StateResult handle_plaintext_state (
     case -1:
       return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
 static StateResult handle_tag_open_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "<"));
   switch (c) {
     case '!':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_MARKUP_DECLARATION);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_MARKUP_DECLARATION_OPEN);
       clear_temporary_buffer(parser);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '/':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_END_TAG_OPEN);
-      append_char_to_temporary_buffer(parser, '/');
-      return NEXT_CHAR;
+      return CONTINUE;
     case '?':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME);
       clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, '?');
-      tokenizer_add_parse_error(parser, GUMBO_ERR_TAG_STARTS_WITH_QUESTION);
-      return NEXT_CHAR;
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_BEFORE_TAG_NAME);
+      // Switch to data to emit EOF.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_from_mark(parser, output);
     default:
       if (is_alpha(c)) {
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_TAG_NAME);
+        reconsume_in_state(parser, GUMBO_LEX_TAG_NAME);
         start_new_tag(parser, true);
-        return NEXT_CHAR;
-      } else {
-        tokenizer_add_parse_error(parser, GUMBO_ERR_TAG_INVALID);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-        emit_temporary_buffer(parser, output);
-        return RETURN_ERROR;
+        return CONTINUE;
       }
+      tokenizer_add_parse_error(parser, GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_from_mark(parser, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
 static StateResult handle_end_tag_open_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "</"));
   switch (c) {
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_CLOSE_TAG_EMPTY);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_END_TAG_NAME);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_CLOSE_TAG_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return emit_temporary_buffer(parser, output);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_BEFORE_TAG_NAME);
+      // Similar to the tag open state except we need to emit '<' and '/'
+      // before the EOF.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_from_mark(parser, output);
     default:
       if (is_alpha(c)) {
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_TAG_NAME);
+        reconsume_in_state(parser, GUMBO_LEX_TAG_NAME);
         start_new_tag(parser, false);
       } else {
-        tokenizer_add_parse_error(parser, GUMBO_ERR_CLOSE_TAG_INVALID);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+        tokenizer_add_parse_error(parser, GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME);
+        reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
         clear_temporary_buffer(parser);
-        append_char_to_temporary_buffer(parser, c);
       }
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -1179,27 +1124,26 @@ static StateResult handle_tag_name_state (
     case ' ':
       finish_tag_name(parser);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '/':
       finish_tag_name(parser);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       finish_tag_name(parser);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       return emit_current_tag(parser, output);
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_TAG_EOF);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      return emit_eof(parser, output);
     default:
       append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -1210,36 +1154,29 @@ static StateResult handle_rcdata_lt_state (
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "<"));
   if (c == '/') {
     gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA_END_TAG_OPEN);
-    append_char_to_temporary_buffer(parser, '/');
-    return NEXT_CHAR;
+    return CONTINUE;
   } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA);
-    tokenizer->_reconsume_current_input = true;
-    return emit_temporary_buffer(parser, output);
+    reconsume_in_state(parser, GUMBO_LEX_RCDATA);
+    return emit_from_mark(parser, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-open-state
 static StateResult handle_rcdata_end_tag_open_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "</"));
   if (is_alpha(c)) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA_END_TAG_NAME);
+    reconsume_in_state(parser, GUMBO_LEX_RCDATA_END_TAG_NAME);
     start_new_tag(parser, false);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA);
-    return emit_temporary_buffer(parser, output);
+    return CONTINUE;
   }
-  return true;
+  reconsume_in_state(parser, GUMBO_LEX_RCDATA);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-name-state
@@ -1250,33 +1187,39 @@ static StateResult handle_rcdata_end_tag_name_state (
   GumboToken* output
 ) {
   UNUSED_IF_NDEBUG(tokenizer);
-  assert(tokenizer->_temporary_buffer.length >= 2);
   if (is_alpha(c)) {
     append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else if (is_appropriate_end_tag(parser)) {
-    switch (c) {
-      case '\t':
-      case '\n':
-      case '\f':
-      case ' ':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-        return NEXT_CHAR;
-      case '/':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-        return NEXT_CHAR;
-      case '>':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-        return emit_current_tag(parser, output);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
     }
+    break;
   }
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA);
   abandon_current_tag(parser);
-  return emit_temporary_buffer(parser, output);
+  reconsume_in_state(parser, GUMBO_LEX_RCDATA);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-less-than-sign-state
@@ -1286,34 +1229,29 @@ static StateResult handle_rawtext_lt_state (
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "<"));
   if (c == '/') {
     gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT_END_TAG_OPEN);
-    append_char_to_temporary_buffer(parser, '/');
-    return NEXT_CHAR;
+    return CONTINUE;
   } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT);
-    tokenizer->_reconsume_current_input = true;
-    return emit_temporary_buffer(parser, output);
+    reconsume_in_state(parser, GUMBO_LEX_RAWTEXT);
+    return emit_from_mark(parser, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-open-state
 static StateResult handle_rawtext_end_tag_open_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "</"));
   if (is_alpha(c)) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT_END_TAG_NAME);
+    reconsume_in_state(parser, GUMBO_LEX_RAWTEXT_END_TAG_NAME);
     start_new_tag(parser, false);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
+    return CONTINUE;
   } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT);
-    return emit_temporary_buffer(parser, output);
+    reconsume_in_state(parser, GUMBO_LEX_RAWTEXT);
+    return emit_from_mark(parser, output);
   }
 }
 
@@ -1324,153 +1262,156 @@ static StateResult handle_rawtext_end_tag_name_state (
   int c,
   GumboToken* output
 ) {
-  assert(tokenizer->_temporary_buffer.length >= 2);
-  gumbo_debug("Last end tag: %*s\n", (int) tokenizer->_tag_state._buffer.length,
-      tokenizer->_tag_state._buffer.data);
   if (is_alpha(c)) {
     append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else if (is_appropriate_end_tag(parser)) {
-    gumbo_debug("Is an appropriate end tag.\n");
-    switch (c) {
-      case '\t':
-      case '\n':
-      case '\f':
-      case ' ':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-        return NEXT_CHAR;
-      case '/':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-        return NEXT_CHAR;
-      case '>':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-        return emit_current_tag(parser, output);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
     }
+    break;
   }
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT);
   abandon_current_tag(parser);
-  return emit_temporary_buffer(parser, output);
+  reconsume_in_state(parser, GUMBO_LEX_RAWTEXT);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-less-than-sign-state
-static StateResult handle_script_lt_state (
+static StateResult handle_script_data_lt_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "<"));
   if (c == '/') {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_END_TAG_OPEN);
-    append_char_to_temporary_buffer(parser, '/');
-    return NEXT_CHAR;
-  } else if (c == '!') {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_START);
-    append_char_to_temporary_buffer(parser, '!');
-    return emit_temporary_buffer(parser, output);
-  } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
-    tokenizer->_reconsume_current_input = true;
-    return emit_temporary_buffer(parser, output);
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_END_TAG_OPEN);
+    return CONTINUE;
+  }
+  if (c == '!') {
+    // This is the only place we don't reconsume the input before emitting the
+    // temporary buffer. Since the current position is stored and the current
+    // character is not emitted, we need to advance the input and then
+    // reconsume.
+    utf8iterator_next(&tokenizer->_input);
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_START);
+    return emit_from_mark(parser, output);
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-open-state
-static StateResult handle_script_end_tag_open_state (
+static StateResult handle_script_data_end_tag_open_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "</"));
   if (is_alpha(c)) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_END_TAG_NAME);
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_END_TAG_NAME);
     start_new_tag(parser, false);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
-    return emit_temporary_buffer(parser, output);
+    return CONTINUE;
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-name-state
-static StateResult handle_script_end_tag_name_state (
+static StateResult handle_script_data_end_tag_name_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  UNUSED_IF_NDEBUG(tokenizer);
-  assert(tokenizer->_temporary_buffer.length >= 2);
   if (is_alpha(c)) {
     append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else if (is_appropriate_end_tag(parser)) {
-    switch (c) {
-      case '\t':
-      case '\n':
-      case '\f':
-      case ' ':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-        return NEXT_CHAR;
-      case '/':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-        return NEXT_CHAR;
-      case '>':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-        return emit_current_tag(parser, output);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
     }
+    break;
   }
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
   abandon_current_tag(parser);
-  return emit_temporary_buffer(parser, output);
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-state
-static StateResult handle_script_escaped_start_state (
+static StateResult handle_script_data_escaped_start_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
   if (c == '-') {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_START_DASH);
-    return emit_current_char(parser, output);
-  } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
-    tokenizer->_reconsume_current_input = true;
-    return NEXT_CHAR;
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_START_DASH);
+    return emit_char(parser, c, output);
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return CONTINUE;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-dash-state
-static StateResult handle_script_escaped_start_dash_state (
+static StateResult handle_script_data_escaped_start_dash_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
   if (c == '-') {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH);
-    return emit_current_char(parser, output);
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH);
+    return emit_char(parser, c, output);
   } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
-    tokenizer->_reconsume_current_input = true;
-    return NEXT_CHAR;
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+    return CONTINUE;
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-state
-static StateResult handle_script_escaped_state (
+static StateResult handle_script_data_escaped_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1478,25 +1419,25 @@ static StateResult handle_script_escaped_state (
 ) {
   switch (c) {
     case '-':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_DASH);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH);
+      return emit_char(parser, c, output);
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_LT);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT);
       clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      set_mark(parser);
+      return CONTINUE;
     case '\0':
       return emit_replacement_char(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SCRIPT_EOF);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
       return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-state
-static StateResult handle_script_escaped_dash_state (
+static StateResult handle_script_data_escaped_dash_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1504,28 +1445,27 @@ static StateResult handle_script_escaped_dash_state (
 ) {
   switch (c) {
     case '-':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH);
+      return emit_char(parser, c, output);
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_LT);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT);
       clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      set_mark(parser);
+      return CONTINUE;
     case '\0':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
       return emit_replacement_char(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SCRIPT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-dash-state
-static StateResult handle_script_escaped_dash_dash_state (
+static StateResult handle_script_data_escaped_dash_dash_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1533,113 +1473,107 @@ static StateResult handle_script_escaped_dash_dash_state (
 ) {
   switch (c) {
     case '-':
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_LT);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT);
       clear_temporary_buffer(parser);
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      set_mark(parser);
+      return CONTINUE;
     case '>':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA);
+      return emit_char(parser, c, output);
     case '\0':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
       return emit_replacement_char(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SCRIPT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-less-than-sign-state
-static StateResult handle_script_escaped_lt_state (
+static StateResult handle_script_data_escaped_lt_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "<"));
-  assert(!tokenizer->_script_data_buffer.length);
+  assert(temporary_buffer_is_empty(parser));
   if (c == '/') {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else if (is_alpha(c)) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START);
-    append_char_to_temporary_buffer(parser, c);
-    gumbo_string_buffer_append_codepoint (
-      ensure_lowercase(c),
-      &tokenizer->_script_data_buffer
-    );
-    return emit_temporary_buffer(parser, output);
-  } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
-    return emit_temporary_buffer(parser, output);
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_OPEN);
+    return CONTINUE;
+  }
+  if (is_alpha(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_START);
+    return emit_from_mark(parser, output);
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-open-state
-static StateResult handle_script_escaped_end_tag_open_state (
+static StateResult handle_script_data_escaped_end_tag_open_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  assert(temporary_buffer_equals(parser, "</"));
   if (is_alpha(c)) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME);
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_NAME);
     start_new_tag(parser, false);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
-    return emit_temporary_buffer(parser, output);
+    return CONTINUE;
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-name-state
-static StateResult handle_script_escaped_end_tag_name_state (
+static StateResult handle_script_data_escaped_end_tag_name_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  UNUSED_IF_NDEBUG(tokenizer);
-  assert(tokenizer->_temporary_buffer.length >= 2);
   if (is_alpha(c)) {
     append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-    append_char_to_temporary_buffer(parser, c);
-    return NEXT_CHAR;
-  } else if (is_appropriate_end_tag(parser)) {
-    switch (c) {
-      case '\t':
-      case '\n':
-      case '\f':
-      case ' ':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-        return NEXT_CHAR;
-      case '/':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-        return NEXT_CHAR;
-      case '>':
-        finish_tag_name(parser);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-        return emit_current_tag(parser, output);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
     }
+    break;
   }
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
   abandon_current_tag(parser);
-  return emit_temporary_buffer(parser, output);
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return emit_from_mark(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-start-state
-static StateResult handle_script_double_escaped_start_state (
+static StateResult handle_script_data_double_escaped_start_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
@@ -1656,29 +1590,23 @@ static StateResult handle_script_double_escaped_start_state (
         parser,
         gumbo_string_equals (
           &kScriptTag,
-          (GumboStringPiece*) &tokenizer->_script_data_buffer
+          (GumboStringPiece*) &tokenizer->_temporary_buffer
         )
-        ? GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED
-        : GUMBO_LEX_SCRIPT_ESCAPED
+        ? GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED
+        : GUMBO_LEX_SCRIPT_DATA_ESCAPED
       );
-      return emit_current_char(parser, output);
-    default:
-      if (is_alpha(c)) {
-        gumbo_string_buffer_append_codepoint (
-          ensure_lowercase(c),
-          &tokenizer->_script_data_buffer
-        );
-        return emit_current_char(parser, output);
-      } else {
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_ESCAPED);
-        tokenizer->_reconsume_current_input = true;
-        return NEXT_CHAR;
-      }
+      return emit_char(parser, c, output);
+  }
+  if (is_alpha(c)) {
+    append_char_to_temporary_buffer(parser, ensure_lowercase(c));
+    return emit_char(parser, c, output);
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return CONTINUE;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-state
-static StateResult handle_script_double_escaped_state (
+static StateResult handle_script_data_double_escaped_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1686,24 +1614,23 @@ static StateResult handle_script_double_escaped_state (
 ) {
   switch (c) {
     case '-':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH);
+      return emit_char(parser, c, output);
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT);
+      return emit_char(parser, c, output);
     case '\0':
       return emit_replacement_char(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SCRIPT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
     default:
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-state
-static StateResult handle_script_double_escaped_dash_state (
+static StateResult handle_script_data_double_escaped_dash_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1712,26 +1639,25 @@ static StateResult handle_script_double_escaped_dash_state (
   switch (c) {
     case '-':
       gumbo_tokenizer_set_state(
-          parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH);
-      return emit_current_char(parser, output);
+          parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH);
+      return emit_char(parser, c, output);
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT);
+      return emit_char(parser, c, output);
     case '\0':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
       return emit_replacement_char(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SCRIPT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-dash-state
-static StateResult handle_script_double_escaped_dash_dash_state (
+static StateResult handle_script_data_double_escaped_dash_dash_state (
   GumboParser* parser,
   GumboTokenizerState* UNUSED_ARG(tokenizer),
   int c,
@@ -1739,46 +1665,44 @@ static StateResult handle_script_double_escaped_dash_dash_state (
 ) {
   switch (c) {
     case '-':
-      return emit_current_char(parser, output);
+      return emit_char(parser, c, output);
     case '<':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT);
+      return emit_char(parser, c, output);
     case '>':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA);
+      return emit_char(parser, c, output);
     case '\0':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
       return emit_replacement_char(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SCRIPT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
-      return emit_current_char(parser, output);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_char(parser, c, output);
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-less-than-sign-state
-static StateResult handle_script_double_escaped_lt_state (
+static StateResult handle_script_data_double_escaped_lt_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
   if (c == '/') {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END);
-    gumbo_string_buffer_clear(&tokenizer->_script_data_buffer);
-    return emit_current_char(parser, output);
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_END);
+    clear_temporary_buffer(parser);
+    return emit_char(parser, c, output);
   } else {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
-    tokenizer->_reconsume_current_input = true;
-    return NEXT_CHAR;
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+    return CONTINUE;
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-end-state
-static StateResult handle_script_double_escaped_end_state (
+static StateResult handle_script_data_double_escaped_end_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
@@ -1793,29 +1717,23 @@ static StateResult handle_script_double_escaped_end_state (
     case '>':
       gumbo_tokenizer_set_state(
           parser, gumbo_string_equals(&kScriptTag,
-                      (GumboStringPiece*) &tokenizer->_script_data_buffer)
-                      ? GUMBO_LEX_SCRIPT_ESCAPED
-                      : GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
-      return emit_current_char(parser, output);
-    default:
-      if (is_alpha(c)) {
-        gumbo_string_buffer_append_codepoint (
-          ensure_lowercase(c),
-          &tokenizer->_script_data_buffer
-        );
-        return emit_current_char(parser, output);
-      } else {
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
-        tokenizer->_reconsume_current_input = true;
-        return NEXT_CHAR;
-      }
+                      (GumboStringPiece*) &tokenizer->_temporary_buffer)
+                      ? GUMBO_LEX_SCRIPT_DATA_ESCAPED
+                      : GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_char(parser, c, output);
+  }
+  if (is_alpha(c)) {
+    append_char_to_temporary_buffer(parser, ensure_lowercase(c));
+    return emit_char(parser, c, output);
   }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+  return CONTINUE;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state
 static StateResult handle_before_attr_name_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
@@ -1824,40 +1742,27 @@ static StateResult handle_before_attr_name_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '/':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-      return NEXT_CHAR;
     case '>':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return emit_current_tag(parser, output);
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_NAME);
-      append_char_to_temporary_buffer(parser, 0xfffd);
-      return NEXT_CHAR;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_NAME_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      abandon_current_tag(parser);
-      return NEXT_CHAR;
-    case '"':
-    case '\'':
-    case '<':
+      reconsume_in_state(parser, GUMBO_LEX_AFTER_ATTR_NAME);
+      return CONTINUE;
     case '=':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_NAME_INVALID);
-    // Fall through.
-    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_NAME);
-      append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-      return NEXT_CHAR;
+      append_char_to_tag_buffer(parser, c, true);
+      return CONTINUE;
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_ATTR_NAME);
+      return CONTINUE;
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
 static StateResult handle_attr_name_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
@@ -1866,45 +1771,35 @@ static StateResult handle_attr_name_state (
     case '\n':
     case '\f':
     case ' ':
-      finish_attribute_name(parser);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_ATTR_NAME);
-      return NEXT_CHAR;
     case '/':
+    case '>':
+    case -1:
       finish_attribute_name(parser);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-      return NEXT_CHAR;
+      reconsume_in_state(parser, GUMBO_LEX_AFTER_ATTR_NAME);
+      return CONTINUE;
     case '=':
       finish_attribute_name(parser);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_VALUE);
-      return NEXT_CHAR;
-    case '>':
-      finish_attribute_name(parser);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      return emit_current_tag(parser, output);
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
-      return NEXT_CHAR;
-    case -1:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      abandon_current_tag(parser);
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_NAME_EOF);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '"':
     case '\'':
     case '<':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_NAME_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME);
     // Fall through.
     default:
       append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-name-state
 static StateResult handle_after_attr_name_state (
   GumboParser* parser,
-  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
@@ -1913,35 +1808,23 @@ static StateResult handle_after_attr_name_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '/':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '=':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_VALUE);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       return emit_current_tag(parser, output);
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_NAME);
-      append_char_to_temporary_buffer(parser, 0xfffd);
-      return NEXT_CHAR;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_NAME_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      return NEXT_CHAR;
-    case '"':
-    case '\'':
-    case '<':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_NAME_INVALID);
-    // Fall through.
+      return emit_eof(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_NAME);
-      append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
-      return NEXT_CHAR;
+      reconsume_in_state(parser, GUMBO_LEX_ATTR_NAME);
+      return CONTINUE;
   }
 }
 
@@ -1957,45 +1840,22 @@ static StateResult handle_before_attr_value_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '"':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED);
       reset_tag_buffer_start_point(parser);
-      return NEXT_CHAR;
-    case '&':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_UNQUOTED);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED);
       reset_tag_buffer_start_point(parser);
-      return NEXT_CHAR;
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_UNQUOTED);
-      append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
-      return NEXT_CHAR;
-    case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_UNQUOTED_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      abandon_current_tag(parser);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_UNQUOTED_RIGHT_BRACKET);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_ATTRIBUTE_VALUE);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_current_tag(parser, output);
-      return RETURN_ERROR;
-    case '<':
-    case '=':
-    case '`':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_UNQUOTED_EQUALS);
-    // Fall through.
-    default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_UNQUOTED);
-      append_char_to_tag_buffer(parser, c, true);
-      return NEXT_CHAR;
+      return emit_current_tag(parser, output);
   }
+  reconsume_in_state(parser, GUMBO_LEX_ATTR_VALUE_UNQUOTED);
+  return CONTINUE;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-double-quoted-state
@@ -2003,30 +1863,28 @@ static StateResult handle_attr_value_double_quoted_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
-  GumboToken* UNUSED_ARG(output)
+  GumboToken* output
 ) {
   switch (c) {
     case '"':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '&':
-      tokenizer->_tag_state._attr_value_state = tokenizer->_state;
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_tag_buffer(parser, kUtf8ReplacementChar, false);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_DOUBLE_QUOTE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      return emit_eof(parser, output);
     default:
       append_char_to_tag_buffer(parser, c, false);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2035,30 +1893,28 @@ static StateResult handle_attr_value_single_quoted_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
-  GumboToken* UNUSED_ARG(output)
+  GumboToken* output
 ) {
   switch (c) {
     case '\'':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '&':
-      tokenizer->_tag_state._attr_value_state = tokenizer->_state;
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_tag_buffer(parser, kUtf8ReplacementChar, false);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_SINGLE_QUOTE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      return emit_eof(parser, output);
     default:
       append_char_to_tag_buffer(parser, c, false);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2076,89 +1932,35 @@ static StateResult handle_attr_value_unquoted_state (
     case ' ':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
       finish_attribute_value(parser);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '&':
-      tokenizer->_tag_state._attr_value_state = tokenizer->_state;
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_ATTR_VALUE_UNQUOTED;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       finish_attribute_value(parser);
       return emit_current_tag(parser, output);
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_UNQUOTED_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      tokenizer->_reconsume_current_input = true;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      return NEXT_CHAR;
-    case '<':
-    case '=':
+      return emit_eof(parser, output);
     case '"':
     case '\'':
+    case '<':
+    case '=':
     case '`':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_UNQUOTED_EQUALS);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE);
     // Fall through.
     default:
       append_char_to_tag_buffer(parser, c, true);
-      return NEXT_CHAR;
-  }
-}
-
-// https://html.spec.whatwg.org/multipage/parsing.html#character-reference-in-attribute-value-state
-static StateResult handle_char_ref_in_attr_value_state (
-  GumboParser* parser,
-  GumboTokenizerState* tokenizer,
-  int UNUSED_ARG(c),
-  GumboToken* UNUSED_ARG(output)
-) {
-  OneOrTwoCodepoints char_ref;
-  int allowed_char;
-  bool is_unquoted = false;
-  switch (tokenizer->_tag_state._attr_value_state) {
-    case GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED:
-      allowed_char = '"';
-      break;
-    case GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED:
-      allowed_char = '\'';
-      break;
-    case GUMBO_LEX_ATTR_VALUE_UNQUOTED:
-      allowed_char = '>';
-      is_unquoted = true;
-      break;
-    default:
-      // -Wmaybe-uninitialized is a little overzealous here, and doesn't
-      // get that the assert(0) means this codepath will never happen.
-      allowed_char = ' ';
-      assert(0);
+      return CONTINUE;
   }
-
-  // Ignore the status, since we don't have a convenient way of signalling that
-  // a parser error has occurred when the error occurs in the middle of a
-  // multi-state token. We'd need a flag inside the TokenizerState to do this,
-  // but that's a low priority fix.
-  gumbo_consume_char_ref (
-    parser,
-    &tokenizer->_input,
-    allowed_char,
-    true,
-    &char_ref
-  );
-  if (char_ref.first != kGumboNoChar) {
-    tokenizer->_reconsume_current_input = true;
-    append_char_to_tag_buffer(parser, char_ref.first, is_unquoted);
-    if (char_ref.second != kGumboNoChar) {
-      append_char_to_tag_buffer(parser, char_ref.second, is_unquoted);
-    }
-  } else {
-    append_char_to_tag_buffer(parser, '&', is_unquoted);
-  }
-  gumbo_tokenizer_set_state(parser, tokenizer->_tag_state._attr_value_state);
-  return NEXT_CHAR;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-value-quoted-state
@@ -2175,24 +1977,21 @@ static StateResult handle_after_attr_value_quoted_state (
     case '\f':
     case ' ':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '/':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       return emit_current_tag(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_AFTER_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      return emit_eof(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_ATTR_AFTER_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES);
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
   }
 }
 
@@ -2209,15 +2008,13 @@ static StateResult handle_self_closing_start_tag_state (
       tokenizer->_tag_state._is_self_closing = true;
       return emit_current_tag(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SOLIDUS_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
       abandon_current_tag(parser);
-      return NEXT_CHAR;
+      return emit_eof(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_SOLIDUS_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
-      tokenizer->_reconsume_current_input = true;
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG);
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
   }
 }
 
@@ -2228,21 +2025,27 @@ static StateResult handle_bogus_comment_state (
   int c,
   GumboToken* output
 ) {
-  while (c != '>' && c != -1) {
-    if (c == '\0') {
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      c = 0xFFFD;
-    }
+  switch (c) {
+  case '>':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+    return emit_comment(parser, output);
+  case -1:
+    // We need to emit the comment and then the EOF, so reconsume in data
+    // state.
+    reconsume_in_state(parser, GUMBO_LEX_DATA);
+    return emit_comment(parser, output);
+  case '\0':
+    tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+    append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+    return CONTINUE;
+  default:
     append_char_to_temporary_buffer(parser, c);
-    utf8iterator_next(&tokenizer->_input);
-    c = utf8iterator_current(&tokenizer->_input);
+    return CONTINUE;
   }
-  gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-  return emit_comment(parser, output);
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
-static StateResult handle_markup_declaration_state (
+static StateResult handle_markup_declaration_open_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int UNUSED_ARG(c),
@@ -2253,21 +2056,21 @@ static StateResult handle_markup_declaration_state (
       &tokenizer->_input,
       "--",
       sizeof("--") - 1,
-      true
+      /* case sensitive */ true
     )
   ) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_START);
-    tokenizer->_reconsume_current_input = true;
-  } else if (
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_START);
+    return CONTINUE;
+  }
+  if (
     utf8iterator_maybe_consume_match (
       &tokenizer->_input,
       "DOCTYPE",
       sizeof("DOCTYPE") - 1,
-      false
+      /* case sensitive */ false
     )
   ) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DOCTYPE);
-    tokenizer->_reconsume_current_input = true;
+    reconsume_in_state(parser, GUMBO_LEX_DOCTYPE);
     // If we get here, we know we'll eventually emit a doctype token, so now is
     // the time to initialize the doctype strings. (Not in doctype_state_init,
     // since then they'll leak if ownership never gets transferred to the
@@ -2275,24 +2078,35 @@ static StateResult handle_markup_declaration_state (
     tokenizer->_doc_type_state.name = gumbo_strdup("");
     tokenizer->_doc_type_state.public_identifier = gumbo_strdup("");
     tokenizer->_doc_type_state.system_identifier = gumbo_strdup("");
-  } else if (
-    tokenizer->_is_current_node_foreign
-    && utf8iterator_maybe_consume_match (
+    return CONTINUE;
+  }
+  if (
+    utf8iterator_maybe_consume_match (
       &tokenizer->_input,
       "[CDATA[", sizeof("[CDATA[") - 1,
-      true
+      /* case sensitive */ true
     )
   ) {
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA);
-    tokenizer->_is_in_cdata = true;
-    tokenizer->_reconsume_current_input = true;
-  } else {
-    tokenizer_add_parse_error(parser, GUMBO_ERR_DASHES_OR_DOCTYPE);
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_COMMENT);
-    tokenizer->_reconsume_current_input = true;
-    clear_temporary_buffer(parser);
+    if (tokenizer->_is_adjusted_current_node_foreign) {
+      reconsume_in_state(parser, GUMBO_LEX_CDATA_SECTION);
+      tokenizer->_is_in_cdata = true;
+      // Start the token after the <![CDATA[.
+      reset_token_start_point(tokenizer);
+    } else {
+      tokenizer_add_token_parse_error(parser, GUMBO_ERR_CDATA_IN_HTML_CONTENT);
+      clear_temporary_buffer(parser);
+      append_string_to_temporary_buffer (
+        parser,
+        &(const GumboStringPiece) { .data = "[CDATA[", .length = 7 }
+      );
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+    }
+    return CONTINUE;
   }
-  return NEXT_CHAR;
+  tokenizer_add_parse_error(parser, GUMBO_ERR_INCORRECTLY_OPENED_COMMENT);
+  reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+  clear_temporary_buffer(parser);
+  return CONTINUE;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
@@ -2305,26 +2119,14 @@ static StateResult handle_comment_start_state (
   switch (c) {
     case '-':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_START_DASH);
-      return NEXT_CHAR;
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
-    case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
+      return emit_comment(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+      return CONTINUE;
   }
 }
 
@@ -2338,28 +2140,20 @@ static StateResult handle_comment_start_dash_state (
   switch (c) {
     case '-':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END);
-      return NEXT_CHAR;
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
+      return emit_comment(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit the EOF next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
       append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2371,21 +2165,99 @@ static StateResult handle_comment_state (
   GumboToken* output
 ) {
   switch (c) {
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT);
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
     case '-':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END_DASH);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit the EOF token next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
     default:
       append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-state
+static StateResult handle_comment_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '!':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT_BANG);
+    append_char_to_temporary_buffer(parser, c);
+    return CONTINUE;
+  case '<':
+    append_char_to_temporary_buffer(parser, c);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-state
+static StateResult handle_comment_lt_bang_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '-':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT_BANG_DASH);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-state
+static StateResult handle_comment_lt_bang_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '-':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT_BANG_DASH_DASH);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_END_DASH);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-dash-state
+static StateResult handle_comment_lt_bang_dash_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '>':
+  case -1:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_END);
+    return CONTINUE;
+  default:
+    tokenizer_add_parse_error(parser, GUMBO_ERR_NESTED_COMMENT);
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_END);
+    return CONTINUE;
   }
 }
 
@@ -2397,25 +2269,18 @@ static StateResult handle_comment_end_dash_state (
   GumboToken* output
 ) {
   switch (c) {
-    case '-':
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END);
-      return NEXT_CHAR;
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
-    case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
-    default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+  case '-':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END);
+    return CONTINUE;
+  case -1:
+    tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+    // Switch to data to emit EOF next.
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+    return emit_comment(parser, output);
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+    append_char_to_temporary_buffer(parser, '-');
+    return CONTINUE;
   }
 }
 
@@ -2430,35 +2295,22 @@ static StateResult handle_comment_end_state (
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       return emit_comment(parser, output);
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
     case '!':
-      tokenizer_add_parse_error(
-          parser, GUMBO_ERR_COMMENT_BANG_AFTER_DOUBLE_DASH);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END_BANG);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '-':
-      tokenizer_add_parse_error(
-          parser, GUMBO_ERR_COMMENT_DASH_AFTER_DOUBLE_DASH);
       append_char_to_temporary_buffer(parser, '-');
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit EOF next.
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
+      return emit_comment(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
       append_char_to_temporary_buffer(parser, '-');
       append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2475,30 +2327,22 @@ static StateResult handle_comment_end_bang_state (
       append_char_to_temporary_buffer(parser, '-');
       append_char_to_temporary_buffer(parser, '-');
       append_char_to_temporary_buffer(parser, '!');
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       return emit_comment(parser, output);
-    case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, '-');
-      append_char_to_temporary_buffer(parser, '!');
-      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_COMMENT_END_BANG_EOF);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit EOF next.
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_comment(parser, output);
-      return RETURN_ERROR;
+      return emit_comment(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT);
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
       append_char_to_temporary_buffer(parser, '-');
       append_char_to_temporary_buffer(parser, '-');
       append_char_to_temporary_buffer(parser, '!');
-      append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2509,26 +2353,27 @@ static StateResult handle_doctype_state (
   int c,
   GumboToken* output
 ) {
-  assert(!tokenizer->_temporary_buffer.length);
+  assert(temporary_buffer_is_empty(parser));
   switch (c) {
     case '\t':
     case '\n':
     case '\f':
     case ' ':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
-      return NEXT_CHAR;
+      return CONTINUE;
+    case '>':
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      // Switch to data to emit EOF next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_SPACE);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
-      tokenizer->_reconsume_current_input = true;
-      tokenizer->_doc_type_state.force_quirks = true;
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME);
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
+      return CONTINUE;
   }
 }
 
@@ -2544,30 +2389,27 @@ static StateResult handle_before_doctype_name_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DOCTYPE_NAME);
-      tokenizer->_doc_type_state.force_quirks = true;
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_RIGHT_BRACKET);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_NAME);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      // Switch to data to emit EOF next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
     default:
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DOCTYPE_NAME);
-      tokenizer->_doc_type_state.force_quirks = false;
       append_char_to_temporary_buffer(parser, ensure_lowercase(c));
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2586,30 +2428,26 @@ static StateResult handle_doctype_name_state (
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_NAME);
       gumbo_free((void*) tokenizer->_doc_type_state.name);
       finish_temporary_buffer(parser, &tokenizer->_doc_type_state.name);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       gumbo_free((void*) tokenizer->_doc_type_state.name);
       finish_temporary_buffer(parser, &tokenizer->_doc_type_state.name);
-      emit_doctype(parser, output);
-      return RETURN_SUCCESS;
+      return emit_doctype(parser, output);
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       gumbo_free((void*) tokenizer->_doc_type_state.name);
       finish_temporary_buffer(parser, &tokenizer->_doc_type_state.name);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DOCTYPE_NAME);
-      tokenizer->_doc_type_state.force_quirks = false;
       append_char_to_temporary_buffer(parser, ensure_lowercase(c));
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2625,35 +2463,29 @@ static StateResult handle_after_doctype_name_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_doctype(parser, output);
-      return RETURN_SUCCESS;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
       if (utf8iterator_maybe_consume_match(
               &tokenizer->_input, "PUBLIC", sizeof("PUBLIC") - 1, false)) {
-        gumbo_tokenizer_set_state(
-            parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD);
-        tokenizer->_reconsume_current_input = true;
+        reconsume_in_state(parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD);
       } else if (utf8iterator_maybe_consume_match(&tokenizer->_input, "SYSTEM",
                      sizeof("SYSTEM") - 1, false)) {
-        gumbo_tokenizer_set_state(
-            parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD);
-        tokenizer->_reconsume_current_input = true;
+        reconsume_in_state(parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD);
       } else {
         tokenizer_add_parse_error(
-            parser, GUMBO_ERR_DOCTYPE_SPACE_OR_RIGHT_BRACKET);
-        gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+            parser, GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME);
+        reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
         tokenizer->_doc_type_state.force_quirks = true;
       }
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2670,37 +2502,34 @@ static StateResult handle_after_doctype_public_keyword_state (
     case '\f':
     case ' ':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '"':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD);
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD);
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_RIGHT_BRACKET);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return CONTINUE;
   }
 }
 
@@ -2716,35 +2545,32 @@ static StateResult handle_before_doctype_public_id_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '"':
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return CONTINUE;
   }
 }
 
@@ -2759,28 +2585,26 @@ static StateResult handle_doctype_public_id_double_quoted_state (
     case '"':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID);
       finish_doctype_public_id(parser);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_public_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_public_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
       append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2795,28 +2619,26 @@ static StateResult handle_doctype_public_id_single_quoted_state (
     case '\'':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID);
       finish_doctype_public_id(parser);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_public_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_public_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
       append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2834,35 +2656,38 @@ static StateResult handle_after_doctype_public_id_state (
     case ' ':
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_doctype(parser, output);
-      return RETURN_SUCCESS;
+      return emit_doctype(parser, output);
     case '"':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
+      tokenizer_add_parse_error (
+        parser,
+        GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS
+      );
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
+      tokenizer_add_parse_error (
+        parser,
+        GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS
+      );
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      tokenizer->_reconsume_current_input = true;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2878,33 +2703,30 @@ static StateResult handle_between_doctype_public_system_id_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_doctype(parser, output);
-      return RETURN_SUCCESS;
+      return emit_doctype(parser, output);
     case '"':
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return CONTINUE;
   }
 }
 
@@ -2921,36 +2743,34 @@ static StateResult handle_after_doctype_system_keyword_state (
     case '\f':
     case ' ':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '"':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD);
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD);
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -2966,34 +2786,32 @@ static StateResult handle_before_doctype_system_id_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '"':
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\'':
       assert(temporary_buffer_is_empty(parser));
       gumbo_tokenizer_set_state(
           parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
       tokenizer->_doc_type_state.force_quirks = true;
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -3008,28 +2826,26 @@ static StateResult handle_doctype_system_id_double_quoted_state (
     case '"':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID);
       finish_doctype_system_id(parser);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_system_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_system_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
       append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -3044,28 +2860,26 @@ static StateResult handle_doctype_system_id_single_quoted_state (
     case '\'':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID);
       finish_doctype_system_id(parser);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '\0':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_UTF8_NULL);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
       append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_END);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER);
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_system_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
       finish_doctype_system_id(parser);
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
       append_char_to_temporary_buffer(parser, c);
-      return NEXT_CHAR;
+      return CONTINUE;
   }
 }
 
@@ -3081,21 +2895,19 @@ static StateResult handle_after_doctype_system_id_state (
     case '\n':
     case '\f':
     case ' ':
-      return NEXT_CHAR;
+      return CONTINUE;
     case '>':
       gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-      emit_doctype(parser, output);
-      return RETURN_SUCCESS;
+      return emit_doctype(parser, output);
     case -1:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_EOF);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
       tokenizer->_doc_type_state.force_quirks = true;
-      emit_doctype(parser, output);
-      return RETURN_ERROR;
+      return emit_doctype(parser, output);
     default:
-      tokenizer_add_parse_error(parser, GUMBO_ERR_DOCTYPE_INVALID);
-      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
-      return NEXT_CHAR;
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      return CONTINUE;
   }
 }
 
@@ -3106,33 +2918,370 @@ static StateResult handle_bogus_doctype_state (
   int c,
   GumboToken* output
 ) {
-  if (c == '>' || c == -1) {
+  switch (c) {
+  case '>':
     gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-    emit_doctype(parser, output);
-    return RETURN_ERROR;
+    return emit_doctype(parser, output);
+  case '\0':
+    tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+    return CONTINUE;
+  case -1:
+    reconsume_in_state(parser, GUMBO_LEX_DATA);
+    return emit_doctype(parser, output);
+  default:
+    return CONTINUE;
   }
-  return NEXT_CHAR;
 }
 
 // https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-state
-static StateResult handle_cdata_state (
+static StateResult handle_cdata_section_state (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
   int c,
   GumboToken* output
 ) {
-  if (c == -1 || utf8iterator_maybe_consume_match(
-                     &tokenizer->_input, "]]>", sizeof("]]>") - 1, true)) {
-    tokenizer->_reconsume_current_input = true;
+  switch (c) {
+  case ']':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA_SECTION_BRACKET);
+    set_mark(parser);
+    return CONTINUE;
+  case -1:
+    tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_CDATA);
+    return emit_eof(parser, output);
+  default:
+    return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-bracket-state
+static StateResult handle_cdata_section_bracket_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case ']':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA_SECTION_END);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_CDATA_SECTION);
+    // Emit the ].
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-end-state
+static StateResult handle_cdata_section_end_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case ']':
+  {
+    // XXX: This is terrible. We want to emit a ] corresponding to the first
+    // of the three in a row we've seen. So let's emit one token from the
+    // temporary buffer (which will rewind 3 characters, emit the ] and
+    // advance one). Next, let's clear the temporary buffer which will set the
+    // mark to the middle of the three brackets. Finally, let's move to the
+    // appropriate state.
+    StateResult result = emit_from_mark(parser, output);
+    tokenizer->_resume_pos = NULL;
+    set_mark(parser);
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA_SECTION);
+    return result;
+  }
+  case '>':
+    // We're done with CDATA so move past the >, reset the token start point
+    // to point after the >, and then reconsume in the data state.
+    utf8iterator_next(&tokenizer->_input);
     reset_token_start_point(tokenizer);
-    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+    reconsume_in_state(parser, GUMBO_LEX_DATA);
     tokenizer->_is_in_cdata = false;
-    return NEXT_CHAR;
-  } else {
-    return emit_current_char(parser, output);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_CDATA_SECTION);
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state
+static StateResult handle_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isalnum(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_NAMED_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  if (c == '#') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+static StateResult handle_named_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  const char *cur = utf8iterator_get_char_pointer(&tokenizer->_input);
+  const char *end = utf8iterator_get_end_pointer(&tokenizer->_input);
+  int code_point[2];
+  size_t size = match_named_char_ref(cur, end - cur, code_point);
+
+  if (size > 0) {
+    utf8iterator_maybe_consume_match(&tokenizer->_input, cur, size, true);
+    int next = utf8iterator_current(&tokenizer->_input);
+    reconsume_in_state(parser, tokenizer->_return_state);
+    if (character_reference_part_of_attribute(parser)
+        && cur[size-1] != ';'
+        && (next == '=' || gumbo_ascii_isalnum(next))) {
+      GumboStringPiece str = { .data = cur, .length = size };
+      append_string_to_temporary_buffer(parser, &str);
+      return flush_code_points_consumed_as_character_reference(parser, output);
+    }
+    if (cur[size-1] != ';')
+      tokenizer_add_char_ref_error(parser, GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE, -1);
+    reconsume_in_state(parser, tokenizer->_return_state);
+    return flush_char_ref(parser, code_point[0], code_point[1], output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_AMBIGUOUS_AMPERSAND);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#ambiguous-ampersand-state
+static StateResult handle_ambiguous_ampersand_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isalnum(c)) {
+    if (character_reference_part_of_attribute(parser)) {
+      append_char_to_tag_buffer(parser, c, true);
+      return CONTINUE;
+    }
+    return emit_char(parser, c, output);
+  }
+  if (c == ';') {
+      tokenizer_add_char_ref_error(parser, GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE, -1);
+    // fall through
+  }
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-state
+static StateResult handle_numeric_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  tokenizer->_character_reference_code = 0;
+  switch (c) {
+  case 'x':
+  case 'X':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE_START);
+    return CONTINUE;
   }
 }
 
+// https://html.spec.whatwg.org/multipage/parsing.html#hexademical-character-reference-start-state
+static StateResult handle_hexadecimal_character_reference_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isxdigit(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error (
+    parser,
+    GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE,
+    -1
+  );
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state
+static StateResult handle_decimal_character_reference_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isdigit(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error (
+    parser,
+    GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE,
+    -1
+  );
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#hexademical-character-reference-state
+static StateResult handle_hexadecimal_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 16 + (c - 0x0030);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (gumbo_ascii_isupper_xdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 16 + (c - 0x0037);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (gumbo_ascii_islower_xdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 16 + (c - 0x0057);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (c == ';') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error(
+    parser,
+    GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE,
+    tokenizer->_character_reference_code
+  );
+  reconsume_in_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-state
+static StateResult handle_decimal_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 10 + (c - 0x0030);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (c == ';') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error(
+    parser,
+    GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE,
+    tokenizer->_character_reference_code
+  );
+  reconsume_in_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
+static StateResult handle_numeric_character_reference_end_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  c = tokenizer->_character_reference_code;
+  if (c == 0) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_NULL_CHARACTER_REFERENCE,
+      c
+    );
+    c = kUtf8ReplacementChar;
+  } else if (c > kUtf8MaxChar) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE,
+      c
+    );
+    c = kUtf8ReplacementChar;
+  } else if (utf8_is_surrogate(c)) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE,
+      c
+    );
+    c = kUtf8ReplacementChar;
+  } else if (utf8_is_noncharacter(c)) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE,
+      c
+    );
+  } else if (c == 0x0D || (utf8_is_control(c) && !gumbo_ascii_isspace(c))) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_CONTROL_CHARACTER_REFERENCE,
+      c
+    );
+    switch (c) {
+    case 0x80: c = 0x20AC; break;
+    case 0x82: c = 0x201A; break;
+    case 0x83: c = 0x0192; break;
+    case 0x84: c = 0x201E; break;
+    case 0x85: c = 0x2026; break;
+    case 0x86: c = 0x2020; break;
+    case 0x87: c = 0x2021; break;
+    case 0x88: c = 0x02C6; break;
+    case 0x89: c = 0x2030; break;
+    case 0x8A: c = 0x0160; break;
+    case 0x8B: c = 0x2039; break;
+    case 0x8C: c = 0x0152; break;
+    case 0x8E: c = 0x017D; break;
+    case 0x91: c = 0x2018; break;
+    case 0x92: c = 0x2019; break;
+    case 0x93: c = 0x201C; break;
+    case 0x94: c = 0x201D; break;
+    case 0x95: c = 0x2022; break;
+    case 0x96: c = 0x2013; break;
+    case 0x97: c = 0x2014; break;
+    case 0x98: c = 0x02DC; break;
+    case 0x99: c = 0x2122; break;
+    case 0x9A: c = 0x0161; break;
+    case 0x9B: c = 0x203A; break;
+    case 0x9C: c = 0x0153; break;
+    case 0x9E: c = 0x017E; break;
+    case 0x9F: c = 0x0178; break;
+    }
+  }
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_char_ref(parser, c, kGumboNoChar, output);
+}
+
 typedef StateResult (*GumboLexerStateFunction) (
   GumboParser* parser,
   GumboTokenizerState* tokenizer,
@@ -3141,77 +3290,89 @@ typedef StateResult (*GumboLexerStateFunction) (
 );
 
 static GumboLexerStateFunction dispatch_table[] = {
-  handle_data_state,
-  handle_char_ref_in_data_state,
-  handle_rcdata_state,
-  handle_char_ref_in_rcdata_state,
-  handle_rawtext_state,
-  handle_script_state,
-  handle_plaintext_state,
-  handle_tag_open_state,
-  handle_end_tag_open_state,
-  handle_tag_name_state,
-  handle_rcdata_lt_state,
-  handle_rcdata_end_tag_open_state,
-  handle_rcdata_end_tag_name_state,
-  handle_rawtext_lt_state,
-  handle_rawtext_end_tag_open_state,
-  handle_rawtext_end_tag_name_state,
-  handle_script_lt_state,
-  handle_script_end_tag_open_state,
-  handle_script_end_tag_name_state,
-  handle_script_escaped_start_state,
-  handle_script_escaped_start_dash_state,
-  handle_script_escaped_state,
-  handle_script_escaped_dash_state,
-  handle_script_escaped_dash_dash_state,
-  handle_script_escaped_lt_state,
-  handle_script_escaped_end_tag_open_state,
-  handle_script_escaped_end_tag_name_state,
-  handle_script_double_escaped_start_state,
-  handle_script_double_escaped_state,
-  handle_script_double_escaped_dash_state,
-  handle_script_double_escaped_dash_dash_state,
-  handle_script_double_escaped_lt_state,
-  handle_script_double_escaped_end_state,
-  handle_before_attr_name_state,
-  handle_attr_name_state,
-  handle_after_attr_name_state,
-  handle_before_attr_value_state,
-  handle_attr_value_double_quoted_state,
-  handle_attr_value_single_quoted_state,
-  handle_attr_value_unquoted_state,
-  handle_char_ref_in_attr_value_state,
-  handle_after_attr_value_quoted_state,
-  handle_self_closing_start_tag_state,
-  handle_bogus_comment_state,
-  handle_markup_declaration_state,
-  handle_comment_start_state,
-  handle_comment_start_dash_state,
-  handle_comment_state,
-  handle_comment_end_dash_state,
-  handle_comment_end_state,
-  handle_comment_end_bang_state,
-  handle_doctype_state,
-  handle_before_doctype_name_state,
-  handle_doctype_name_state,
-  handle_after_doctype_name_state,
-  handle_after_doctype_public_keyword_state,
-  handle_before_doctype_public_id_state,
-  handle_doctype_public_id_double_quoted_state,
-  handle_doctype_public_id_single_quoted_state,
-  handle_after_doctype_public_id_state,
-  handle_between_doctype_public_system_id_state,
-  handle_after_doctype_system_keyword_state,
-  handle_before_doctype_system_id_state,
-  handle_doctype_system_id_double_quoted_state,
-  handle_doctype_system_id_single_quoted_state,
-  handle_after_doctype_system_id_state,
-  handle_bogus_doctype_state,
-  handle_cdata_state
+  [GUMBO_LEX_DATA] = handle_data_state,
+  [GUMBO_LEX_RCDATA] = handle_rcdata_state,
+  [GUMBO_LEX_RAWTEXT] = handle_rawtext_state,
+  [GUMBO_LEX_SCRIPT_DATA] = handle_script_data_state,
+  [GUMBO_LEX_PLAINTEXT] = handle_plaintext_state,
+  [GUMBO_LEX_TAG_OPEN] = handle_tag_open_state,
+  [GUMBO_LEX_END_TAG_OPEN] = handle_end_tag_open_state,
+  [GUMBO_LEX_TAG_NAME] = handle_tag_name_state,
+  [GUMBO_LEX_RCDATA_LT] = handle_rcdata_lt_state,
+  [GUMBO_LEX_RCDATA_END_TAG_OPEN] = handle_rcdata_end_tag_open_state,
+  [GUMBO_LEX_RCDATA_END_TAG_NAME] = handle_rcdata_end_tag_name_state,
+  [GUMBO_LEX_RAWTEXT_LT] = handle_rawtext_lt_state,
+  [GUMBO_LEX_RAWTEXT_END_TAG_OPEN] = handle_rawtext_end_tag_open_state,
+  [GUMBO_LEX_RAWTEXT_END_TAG_NAME] = handle_rawtext_end_tag_name_state,
+  [GUMBO_LEX_SCRIPT_DATA_LT] = handle_script_data_lt_state,
+  [GUMBO_LEX_SCRIPT_DATA_END_TAG_OPEN] = handle_script_data_end_tag_open_state,
+  [GUMBO_LEX_SCRIPT_DATA_END_TAG_NAME] = handle_script_data_end_tag_name_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_START] = handle_script_data_escaped_start_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_START_DASH] = handle_script_data_escaped_start_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED] = handle_script_data_escaped_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH] = handle_script_data_escaped_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH] = handle_script_data_escaped_dash_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT] = handle_script_data_escaped_lt_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_OPEN] = handle_script_data_escaped_end_tag_open_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_NAME] = handle_script_data_escaped_end_tag_name_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_START] = handle_script_data_double_escaped_start_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED] = handle_script_data_double_escaped_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH] = handle_script_data_double_escaped_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH] = handle_script_data_double_escaped_dash_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT] = handle_script_data_double_escaped_lt_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_END] = handle_script_data_double_escaped_end_state,
+  [GUMBO_LEX_BEFORE_ATTR_NAME] = handle_before_attr_name_state,
+  [GUMBO_LEX_ATTR_NAME] = handle_attr_name_state,
+  [GUMBO_LEX_AFTER_ATTR_NAME] = handle_after_attr_name_state,
+  [GUMBO_LEX_BEFORE_ATTR_VALUE] = handle_before_attr_value_state,
+  [GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED] = handle_attr_value_double_quoted_state,
+  [GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED] = handle_attr_value_single_quoted_state,
+  [GUMBO_LEX_ATTR_VALUE_UNQUOTED] = handle_attr_value_unquoted_state,
+  [GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED] = handle_after_attr_value_quoted_state,
+  [GUMBO_LEX_SELF_CLOSING_START_TAG] = handle_self_closing_start_tag_state,
+  [GUMBO_LEX_BOGUS_COMMENT] = handle_bogus_comment_state,
+  [GUMBO_LEX_MARKUP_DECLARATION_OPEN] = handle_markup_declaration_open_state,
+  [GUMBO_LEX_COMMENT_START] = handle_comment_start_state,
+  [GUMBO_LEX_COMMENT_START_DASH] = handle_comment_start_dash_state,
+  [GUMBO_LEX_COMMENT] = handle_comment_state,
+  [GUMBO_LEX_COMMENT_LT] = handle_comment_lt_state,
+  [GUMBO_LEX_COMMENT_LT_BANG] = handle_comment_lt_bang_state,
+  [GUMBO_LEX_COMMENT_LT_BANG_DASH] = handle_comment_lt_bang_dash_state,
+  [GUMBO_LEX_COMMENT_LT_BANG_DASH_DASH] = handle_comment_lt_bang_dash_dash_state,
+  [GUMBO_LEX_COMMENT_END_DASH] = handle_comment_end_dash_state,
+  [GUMBO_LEX_COMMENT_END] = handle_comment_end_state,
+  [GUMBO_LEX_COMMENT_END_BANG] = handle_comment_end_bang_state,
+  [GUMBO_LEX_DOCTYPE] = handle_doctype_state,
+  [GUMBO_LEX_BEFORE_DOCTYPE_NAME] = handle_before_doctype_name_state,
+  [GUMBO_LEX_DOCTYPE_NAME] = handle_doctype_name_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_NAME] = handle_after_doctype_name_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD] = handle_after_doctype_public_keyword_state,
+  [GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID] = handle_before_doctype_public_id_state,
+  [GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED] = handle_doctype_public_id_double_quoted_state,
+  [GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED] = handle_doctype_public_id_single_quoted_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID] = handle_after_doctype_public_id_state,
+  [GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID] = handle_between_doctype_public_system_id_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD] = handle_after_doctype_system_keyword_state,
+  [GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID] = handle_before_doctype_system_id_state,
+  [GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED] = handle_doctype_system_id_double_quoted_state,
+  [GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED] = handle_doctype_system_id_single_quoted_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID] = handle_after_doctype_system_id_state,
+  [GUMBO_LEX_BOGUS_DOCTYPE] = handle_bogus_doctype_state,
+  [GUMBO_LEX_CDATA_SECTION] = handle_cdata_section_state,
+  [GUMBO_LEX_CDATA_SECTION_BRACKET] = handle_cdata_section_bracket_state,
+  [GUMBO_LEX_CDATA_SECTION_END] = handle_cdata_section_end_state,
+  [GUMBO_LEX_CHARACTER_REFERENCE] = handle_character_reference_state,
+  [GUMBO_LEX_NAMED_CHARACTER_REFERENCE] = handle_named_character_reference_state,
+  [GUMBO_LEX_AMBIGUOUS_AMPERSAND] = handle_ambiguous_ampersand_state,
+  [GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE] = handle_numeric_character_reference_state,
+  [GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START] = handle_hexadecimal_character_reference_start_state,
+  [GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE_START] = handle_decimal_character_reference_start_state,
+  [GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE] = handle_hexadecimal_character_reference_state,
+  [GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE] = handle_decimal_character_reference_state,
+  [GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END] = handle_numeric_character_reference_end_state,
 };
 
-bool gumbo_lex(GumboParser* parser, GumboToken* output) {
+void gumbo_lex(GumboParser* parser, GumboToken* output) {
   // Because of the spec requirements that...
   //
   // 1. Tokens be handled immediately by the parser upon emission.
@@ -3236,15 +3397,15 @@ bool gumbo_lex(GumboParser* parser, GumboToken* output) {
     // isn't consumed twice.
     tokenizer->_reconsume_current_input = false;
     tokenizer->_buffered_emit_char = kGumboNoChar;
-    return true;
+    return;
   }
 
-  if (maybe_emit_from_temporary_buffer(parser, output)) {
-    return true;
+  if (maybe_emit_from_mark(parser, output) == EMIT_TOKEN) {
+    return;
   }
 
   while (1) {
-    assert(!tokenizer->_temporary_buffer_emit);
+    assert(!tokenizer->_resume_pos);
     assert(tokenizer->_buffered_emit_char == kGumboNoChar);
     int c = utf8iterator_current(&tokenizer->_input);
     GumboTokenizerEnum state = tokenizer->_state;
@@ -3255,11 +3416,8 @@ bool gumbo_lex(GumboParser* parser, GumboToken* output) {
     bool should_advance = !tokenizer->_reconsume_current_input;
     tokenizer->_reconsume_current_input = false;
 
-    if (result == RETURN_SUCCESS) {
-      return true;
-    } else if (result == RETURN_ERROR) {
-      return false;
-    }
+    if (result == EMIT_TOKEN)
+      return;
 
     if (should_advance) {
       utf8iterator_next(&tokenizer->_input);
@@ -3285,12 +3443,16 @@ void gumbo_token_destroy(GumboToken* token) {
         }
       }
       gumbo_free((void*) token->v.start_tag.attributes.data);
-      if (token->v.start_tag.tag == GUMBO_TAG_UNKNOWN)
+      if (token->v.start_tag.tag == GUMBO_TAG_UNKNOWN) {
         gumbo_free(token->v.start_tag.name);
+        token->v.start_tag.name = NULL;
+      }
       return;
     case GUMBO_TOKEN_END_TAG:
-      if (token->v.end_tag.tag == GUMBO_TAG_UNKNOWN)
+      if (token->v.end_tag.tag == GUMBO_TAG_UNKNOWN) {
         gumbo_free(token->v.end_tag.name);
+        token->v.end_tag.name = NULL;
+      }
       break;
     case GUMBO_TOKEN_COMMENT:
       gumbo_free((void*) token->v.text);