nokogiri 1.10.9 → 1.18.3

data/gumbo-parser/src/tokenizer.c

@@ -0,0 +1,3464 @@
+/*
+ Copyright 2010 Google Inc.
+ Copyright 2017-2018 Craig Barnes
+ Copyright 2018 Stephen Checkoway
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+/*
+ Coding conventions specific to this file:
+
+ 1. Functions that fill in a token should be named emit_*, and should be
+    followed immediately by a return from the tokenizer.
+ 2. Functions that shuffle data from temporaries to final API structures
+    should be named finish_*, and be called just before the tokenizer exits the
+    state that accumulates the temporary.
+ 3. All internal data structures should be kept in an initialized state from
+    tokenizer creation onwards, ready to accept input. When a buffer's flushed
+    and reset, it should be deallocated and immediately reinitialized.
+ 4. Make sure there are appropriate break statements following each state.
+ 5. Assertions on the state of the temporary and tag buffers are usually a
+    good idea, and should go at the entry point of each state when added.
+ 6. Statement order within states goes:
+    1. Add parse errors, if appropriate.
+    2. Call finish_* functions to build up tag state.
+    2. Switch to new state. Set _reconsume flag if appropriate.
+    3. Perform any other temporary buffer manipulation.
+    4. Emit tokens
+    5. Return/break.
+    This order ensures that we can verify that every emit is followed by
+    a return, ensures that the correct state is recorded with any parse
+    errors, and prevents parse error position from being messed up by
+    possible mark/resets in temporary buffer manipulation.
+*/
+
+#include <assert.h>
+#include <string.h>
+#include "tokenizer.h"
+#include "ascii.h"
+#include "attribute.h"
+#include "char_ref.h"
+#include "error.h"
+#include "nokogiri_gumbo.h"
+#include "parser.h"
+#include "string_buffer.h"
+#include "token_type.h"
+#include "tokenizer_states.h"
+#include "utf8.h"
+#include "util.h"
+#include "vector.h"
+
+// Compared against _temporary_buffer to determine if we're in
+// double-escaped script mode.
+static const GumboStringPiece kScriptTag = {.data = "script", .length = 6};
+
+// An enum for the return value of each individual state. Each of the emit_*
+// functions should return EMIT_TOKEN and should be called as
+// return emit_foo(parser, ..., output);
+// Each of the handle_*_state functions that do not return emit_* should
+// instead return CONTINUE to indicate to gumbo_lex to continue lexing.
+typedef enum {
+  EMIT_TOKEN,
+  CONTINUE,
+} StateResult;
+
+// This is a struct containing state necessary to build up a tag token,
+// character by character.
+typedef struct GumboInternalTagState {
+  // A buffer to accumulate characters for various GumboStringPiece fields.
+  GumboStringBuffer _buffer;
+
+  // A pointer to the start of the original text corresponding to the contents
+  // of the buffer.
+  const char* _original_text;
+
+  // The current tag enum, computed once the tag name state has finished so that
+  // the buffer can be re-used for building up attributes.
+  GumboTag _tag;
+
+  // The current tag name. It's set at the same time that _tag is set if _tag
+  // is set to GUMBO_TAG_UNKNOWN.
+  char *_name;
+
+  // The starting location of the text in the buffer.
+  GumboSourcePosition _start_pos;
+
+  // The current list of attributes. This is copied (and ownership of its data
+  // transferred) to the GumboStartTag token upon completion of the tag. New
+  // attributes are added as soon as their attribute name state is complete, and
+  // values are filled in by operating on _attributes.data[attributes.length-1].
+  GumboVector /* GumboAttribute */ _attributes;
+
+  // If true, the next attribute value to be finished should be dropped. This
+  // happens if a duplicate attribute name is encountered - we want to consume
+  // the attribute value, but shouldn't overwrite the existing value.
+  bool _drop_next_attr_value;
+
+  // The last start tag to have been emitted by the tokenizer. This is
+  // necessary to check for appropriate end tags.
+  GumboTag _last_start_tag;
+
+  // If true, then this is a start tag. If false, it's an end tag. This is
+  // necessary to generate the appropriate token type at tag-closing time.
+  bool _is_start_tag;
+
+  // If true, then this tag is "self-closing" and doesn't have an end tag.
+  bool _is_self_closing;
+} GumboTagState;
+
+// This is the main tokenizer state struct, containing all state used by in
+// tokenizing the input stream.
+typedef struct GumboInternalTokenizerState {
+  // The current lexer state. Starts in GUMBO_LEX_DATA.
+  GumboTokenizerEnum _state;
+
+  // A flag indicating whether the current input character needs to reconsumed
+  // in another state, or whether the next input character should be read for
+  // the next iteration of the state loop. This is set when the spec reads
+  // "Reconsume the current input character in..."
+  bool _reconsume_current_input;
+
+  // A flag indicating whether the adjusted current node is a foreign element.
+  // This is set by gumbo_tokenizer_set_is_adjusted_current_node_foreign and
+  // checked in the markup declaration state.
+  bool _is_adjusted_current_node_foreign;
+
+  // A flag indicating whether the tokenizer is in a CDATA section. If so, then
+  // text tokens emitted will be GUMBO_TOKEN_CDATA.
+  bool _is_in_cdata;
+
+  // Certain states (notably character references) may emit two character tokens
+  // at once, but the contract for lex() fills in only one token at a time. The
+  // extra character is buffered here, and then this is checked on entry to
+  // lex(). If a character is stored here, it's immediately emitted and control
+  // returns from the lexer. kGumboNoChar is used to represent 'no character
+  // stored.'
+  //
+  // Note that characters emitted through this mechanism will have their source
+  // position marked as the character under the mark, i.e. multiple characters
+  // may be emitted with the same position. This is desirable for character
+  // references, but unsuitable for many other cases. Use the _temporary_buffer
+  // mechanism if the buffered characters must have their original positions in
+  // the document.
+  int _buffered_emit_char;
+
+  // A temporary buffer to accumulate characters, as described by the "temporary
+  // buffer" phrase in the tokenizer spec. We use this in a somewhat unorthodox
+  // way: In situations where the spec calls for inserting characters into the
+  // temporary buffer that exactly match the input in order to emit them as
+  // character tokens, we don't actually do it.
+  // Instead, we mark the input and reset the input to it using set_mark() and
+  // emit_from_mark(). We do use the temporary buffer for other uses such as
+  // DOCTYPEs, comments, and detecting escaped <script> tags.
+  GumboStringBuffer _temporary_buffer;
+
+  // The position to resume normal operation after we start emitting from the
+  // mark. NULL whenever we're not emitting from the mark.
+  const char* _resume_pos;
+
+  // The character reference state uses a return state to return to the state
+  // it was invoked from.
+  GumboTokenizerEnum _return_state;
+
+  // Numeric character reference.
+  uint32_t _character_reference_code;
+
+  // Pointer to the beginning of the current token in the original buffer; used
+  // to record the original text.
+  const char* _token_start;
+
+  // GumboSourcePosition recording the source location of the start of the
+  // current token.
+  GumboSourcePosition _token_start_pos;
+
+  // Current tag state.
+  GumboTagState _tag_state;
+
+  // Doctype state. We use the temporary buffer to accumulate characters (it's
+  // not used for anything else in the doctype states), and then freshly
+  // allocate the strings in the doctype token, then copy it over on emit.
+  GumboTokenDocType _doc_type_state;
+
+  // The UTF8Iterator over the tokenizer input.
+  Utf8Iterator _input;
+} GumboTokenizerState;
+
+// Adds a parse error to the parser's error struct.
+static void tokenizer_add_parse_error (
+  GumboParser* parser,
+  GumboErrorType type
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboError* error = gumbo_add_error(parser);
+  if (!error) {
+    return;
+  }
+  const Utf8Iterator* input = &tokenizer->_input;
+  utf8iterator_get_position(input, &error->position);
+  error->original_text.data = utf8iterator_get_char_pointer(input);
+  error->original_text.length = utf8iterator_get_width(input);
+  error->type = type;
+  error->v.tokenizer.state = tokenizer->_state;
+  error->v.tokenizer.codepoint = utf8iterator_current(input);
+}
+
+// Adds an error pointing at the start of the character reference.
+static void tokenizer_add_char_ref_error (
+  struct GumboInternalParser* parser,
+  GumboErrorType type,
+  int codepoint
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboError* error = gumbo_add_error(parser);
+  if (!error)
+    return;
+  Utf8Iterator* input = &tokenizer->_input;
+  error->type = type;
+  error->position = utf8iterator_get_mark_position(input);
+  const char* mark = utf8iterator_get_mark_pointer(input);
+  error->original_text.data = mark;
+  error->original_text.length = utf8iterator_get_char_pointer(input) - mark;
+  error->v.tokenizer.state = tokenizer->_state;
+  error->v.tokenizer.codepoint = codepoint;
+}
+
+// Adds an error pointing at the start of the token.
+static void tokenizer_add_token_parse_error (
+  GumboParser* parser,
+  GumboErrorType type
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboError* error = gumbo_add_error(parser);
+  if (!error)
+    return;
+  Utf8Iterator* input = &tokenizer->_input;
+  error->type = type;
+  error->position = tokenizer->_token_start_pos;
+  error->original_text.data = tokenizer->_token_start;
+  error->original_text.length =
+    utf8iterator_get_char_pointer(input) - tokenizer->_token_start;
+  error->v.tokenizer.state = tokenizer->_state;
+  error->v.tokenizer.codepoint = 0;
+}
+
+static bool is_alpha(int c) {
+  return gumbo_ascii_isalpha(c);
+}
+
+static int ensure_lowercase(int c) {
+  return gumbo_ascii_tolower(c);
+}
+
+static GumboTokenType get_char_token_type(bool is_in_cdata, int c) {
+  if (is_in_cdata && c > 0) {
+    return GUMBO_TOKEN_CDATA;
+  }
+
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\r':
+    case '\f':
+    case ' ':
+      return GUMBO_TOKEN_WHITESPACE;
+    case 0:
+      gumbo_debug("Emitted null byte.\n");
+      return GUMBO_TOKEN_NULL;
+    case -1:
+      return GUMBO_TOKEN_EOF;
+    default:
+      return GUMBO_TOKEN_CHARACTER;
+  }
+}
+
+// Starts recording characters in the temporary buffer.
+static void clear_temporary_buffer(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  gumbo_string_buffer_clear(&tokenizer->_temporary_buffer);
+}
+
+// Appends a codepoint to the temporary buffer.
+static void append_char_to_temporary_buffer (
+  GumboParser* parser,
+  int codepoint
+) {
+  gumbo_string_buffer_append_codepoint (
+   codepoint,
+   &parser->_tokenizer_state->_temporary_buffer
+  );
+}
+
+static void append_string_to_temporary_buffer (
+  GumboParser* parser,
+  const GumboStringPiece* str
+) {
+  gumbo_string_buffer_append_string (
+    str,
+    &parser->_tokenizer_state->_temporary_buffer
+  );
+}
+
+
+static bool temporary_buffer_is_empty(const GumboParser* parser) {
+  return parser->_tokenizer_state->_temporary_buffer.length == 0;
+}
+
+static void doc_type_state_init(GumboParser* parser) {
+  GumboTokenDocType* doc_type_state =
+      &parser->_tokenizer_state->_doc_type_state;
+  // We initialize these to NULL here so that we don't end up leaking memory if
+  // we never see a doctype token. When we do see a doctype token, we reset
+  // them to a freshly-allocated empty string so that we can present a uniform
+  // interface to client code and not make them check for null. Ownership is
+  // transferred to the doctype token when it's emitted.
+  doc_type_state->name = NULL;
+  doc_type_state->public_identifier = NULL;
+  doc_type_state->system_identifier = NULL;
+  doc_type_state->force_quirks = false;
+  doc_type_state->has_public_identifier = false;
+  doc_type_state->has_system_identifier = false;
+}
+
+// Sets the token original_text and position to the current iterator position.
+// This is necessary because [CDATA[ sections may include text that is ignored
+// by the tokenizer.
+static void reset_token_start_point(GumboTokenizerState* tokenizer) {
+  tokenizer->_token_start = utf8iterator_get_char_pointer(&tokenizer->_input);
+  utf8iterator_get_position(&tokenizer->_input, &tokenizer->_token_start_pos);
+}
+
+// Sets the tag buffer original text and start point to the current iterator
+// position. This is necessary because attribute names & values may have
+// whitespace preceding them, and so we can't assume that the actual token
+// starting point was the end of the last tag buffer usage.
+static void reset_tag_buffer_start_point(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+
+  utf8iterator_get_position(&tokenizer->_input, &tag_state->_start_pos);
+  tag_state->_original_text = utf8iterator_get_char_pointer(&tokenizer->_input);
+}
+
+// Moves the temporary buffer contents over to the specified output string,
+// and clears the temporary buffer.
+static void finish_temporary_buffer(GumboParser* parser, const char** output) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  *output = gumbo_string_buffer_to_string(&tokenizer->_temporary_buffer);
+  clear_temporary_buffer(parser);
+}
+
+// Advances the iterator past the end of the token, and then fills in the
+// relevant position fields. It's assumed that after every emit, the tokenizer
+// will immediately return (letting the tree-construction stage read the filled
+// in Token). Thus, it's safe to advance the input stream here, since it will
+// bypass the advance at the bottom of the state machine loop.
+//
+// Since this advances the iterator and resets the current input, make sure to
+// call it after you've recorded any other data you need for the token.
+static void finish_token(GumboParser* parser, GumboToken* token) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  if (!tokenizer->_reconsume_current_input) {
+    utf8iterator_next(&tokenizer->_input);
+  }
+
+  token->position = tokenizer->_token_start_pos;
+  token->original_text.data = tokenizer->_token_start;
+  reset_token_start_point(tokenizer);
+  token->original_text.length =
+      tokenizer->_token_start - token->original_text.data;
+  if (token->original_text.length > 0 &&
+      token->original_text.data[token->original_text.length - 1] == '\r') {
+    // The UTF8 iterator will ignore carriage returns in the input stream, which
+    // means that the next token may start one past a \r character. The pointer
+    // arithmetic above results in that \r being appended to the original text
+    // of the preceding token, so we have to adjust its length here to chop the
+    // \r off.
+    --token->original_text.length;
+  }
+}
+
+// Records the doctype public ID, assumed to be in the temporary buffer.
+// Convenience method that also sets has_public_identifier to true.
+static void finish_doctype_public_id(GumboParser* parser) {
+  GumboTokenDocType* doc_type_state =
+      &parser->_tokenizer_state->_doc_type_state;
+  gumbo_free((void*) doc_type_state->public_identifier);
+  finish_temporary_buffer(parser, &doc_type_state->public_identifier);
+  doc_type_state->has_public_identifier = true;
+}
+
+// Records the doctype system ID, assumed to be in the temporary buffer.
+// Convenience method that also sets has_system_identifier to true.
+static void finish_doctype_system_id(GumboParser* parser) {
+  GumboTokenDocType* doc_type_state =
+      &parser->_tokenizer_state->_doc_type_state;
+  gumbo_free((void*) doc_type_state->system_identifier);
+  finish_temporary_buffer(parser, &doc_type_state->system_identifier);
+  doc_type_state->has_system_identifier = true;
+}
+
+// Writes a single specified character to the output token.
+static StateResult emit_char(GumboParser* parser, int c, GumboToken* output) {
+  output->type = get_char_token_type(parser->_tokenizer_state->_is_in_cdata, c);
+  output->v.character = c;
+  finish_token(parser, output);
+  return EMIT_TOKEN;
+}
+
+// Writes a replacement character token and records a parse error.
+// Always returns EMIT_TOKEN, per gumbo_lex return value.
+static StateResult emit_replacement_char(
+    GumboParser* parser, GumboToken* output) {
+  // In all cases, this is because of a null byte in the input stream.
+  tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+  emit_char(parser, kUtf8ReplacementChar, output);
+  return EMIT_TOKEN;
+}
+
+// Writes an EOF character token. Always returns EMIT_TOKEN.
+static StateResult emit_eof(GumboParser* parser, GumboToken* output) {
+  return emit_char(parser, -1, output);
+}
+
+// Writes out a doctype token, copying it from the tokenizer state.
+static StateResult emit_doctype(GumboParser* parser, GumboToken* output) {
+  output->type = GUMBO_TOKEN_DOCTYPE;
+  output->v.doc_type = parser->_tokenizer_state->_doc_type_state;
+  finish_token(parser, output);
+  doc_type_state_init(parser);
+  return EMIT_TOKEN;
+}
+
+// Debug-only function that explicitly sets the attribute vector data to NULL so
+// it can be asserted on tag creation, verifying that there are no memory leaks.
+static void mark_tag_state_as_empty(GumboTagState* tag_state) {
+  UNUSED_IF_NDEBUG(tag_state);
+  tag_state->_name = NULL;
+#ifndef NDEBUG
+  tag_state->_attributes = kGumboEmptyVector;
+#endif
+}
+
+// Writes out the current tag as a start or end tag token.
+// Always returns EMIT_TOKEN.
+static StateResult emit_current_tag(GumboParser* parser, GumboToken* output) {
+  GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
+  if (tag_state->_is_start_tag) {
+    output->type = GUMBO_TOKEN_START_TAG;
+    output->v.start_tag.tag = tag_state->_tag;
+    output->v.start_tag.name = tag_state->_name;
+    output->v.start_tag.attributes = tag_state->_attributes;
+    output->v.start_tag.is_self_closing = tag_state->_is_self_closing;
+    tag_state->_last_start_tag = tag_state->_tag;
+    mark_tag_state_as_empty(tag_state);
+    gumbo_debug(
+        "Emitted start tag %s.\n", gumbo_normalized_tagname(tag_state->_tag));
+  } else {
+    output->type = GUMBO_TOKEN_END_TAG;
+    output->v.end_tag.tag = tag_state->_tag;
+    output->v.end_tag.name = tag_state->_name;
+    if (tag_state->_is_self_closing)
+      tokenizer_add_token_parse_error(parser, GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS);
+    if (tag_state->_attributes.length > 0)
+      tokenizer_add_token_parse_error(parser, GUMBO_ERR_END_TAG_WITH_ATTRIBUTES);
+    // In end tags, ownership of the attributes vector is not transferred to the
+    // token, but it's still initialized as normal, so it must be manually
+    // deallocated. There may also be attributes to destroy, in certain broken
+    // cases like </div</th> (the "th" is an attribute there).
+    for (unsigned int i = 0; i < tag_state->_attributes.length; ++i) {
+      gumbo_destroy_attribute(tag_state->_attributes.data[i]);
+    }
+    gumbo_free(tag_state->_attributes.data);
+    mark_tag_state_as_empty(tag_state);
+    gumbo_debug(
+        "Emitted end tag %s.\n", gumbo_normalized_tagname(tag_state->_tag));
+  }
+  gumbo_string_buffer_destroy(&tag_state->_buffer);
+  finish_token(parser, output);
+  gumbo_debug (
+    "Original text = %.*s.\n",
+    (int) output->original_text.length,
+    output->original_text.data
+  );
+  assert(output->original_text.length >= 2);
+  assert(output->original_text.data[0] == '<');
+  assert(output->original_text.data[output->original_text.length - 1] == '>');
+  return EMIT_TOKEN;
+}
+
+// In some states, we speculatively start a tag, but don't know whether it'll be
+// emitted as tag token or as a series of character tokens until we finish it.
+// We need to abandon the tag we'd started & free its memory in that case to
+// avoid a memory leak.
+static void abandon_current_tag(GumboParser* parser) {
+  GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
+  for (unsigned int i = 0; i < tag_state->_attributes.length; ++i) {
+    gumbo_destroy_attribute(tag_state->_attributes.data[i]);
+  }
+  gumbo_free(tag_state->_name);
+  gumbo_free(tag_state->_attributes.data);
+  mark_tag_state_as_empty(tag_state);
+  gumbo_string_buffer_destroy(&tag_state->_buffer);
+  gumbo_debug("Abandoning current tag.\n");
+}
+
+// Emits a comment token. Comments use the temporary buffer to accumulate their
+// data, and then it's copied over and released to the 'text' field of the
+// GumboToken union. Always returns EMIT_TOKEN.
+static StateResult emit_comment(GumboParser* parser, GumboToken* output) {
+  output->type = GUMBO_TOKEN_COMMENT;
+  finish_temporary_buffer(parser, &output->v.text);
+  finish_token(parser, output);
+  return EMIT_TOKEN;
+}
+
+static void set_mark(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  utf8iterator_mark(&tokenizer->_input);
+}
+
+// Checks to see we should be emitting characters from the mark, and fills the
+// output token with the next output character if so.
+// Returns EMIT_TOKEN if a character has been emitted and the tokenizer should
+// immediately return, CONTINUE if we should resume normal operation.
+static StateResult maybe_emit_from_mark (
+    GumboParser* parser,
+    GumboToken* output
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  const char* pos = tokenizer->_resume_pos;
+
+  if (!pos)
+    return CONTINUE;
+  if (utf8iterator_get_char_pointer(&tokenizer->_input) >= pos) {
+    tokenizer->_resume_pos = NULL;
+    return CONTINUE;
+  }
+
+  // emit_char advances the input stream. _reconsume_current_input should
+  // *never* be set when emitting from the mark since those characters have
+  // already been advanced past.
+  assert(!tokenizer->_reconsume_current_input);
+  return emit_char(parser, utf8iterator_current(&tokenizer->_input), output);
+}
+
+// Sets up the tokenizer to begin emitting from the mark up to, but not
+// including, the current code point. This resets the input iterator stream to
+// the mark, sets up _resume_pos, and then emits the first character in it.
+// Returns EMIT_TOKEN.
+static StateResult emit_from_mark(GumboParser* parser, GumboToken* output) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  tokenizer->_resume_pos = utf8iterator_get_char_pointer(&tokenizer->_input);
+  utf8iterator_reset(&tokenizer->_input);
+  // Now that we have reset the input, we need to advance through it.
+  tokenizer->_reconsume_current_input = false;
+  StateResult result = maybe_emit_from_mark(parser, output);
+  assert(result == EMIT_TOKEN);
+  return result;
+}
+
+// Appends a codepoint to the current tag buffer. If
+// reinitialize_position_on_first is set, this also initializes the tag buffer
+// start point; the only time you would *not* want to pass true for this
+// parameter is if you want the original_text to include character (like an
+// opening quote) that doesn't appear in the value.
+static void append_char_to_tag_buffer (
+  GumboParser* parser,
+  int codepoint,
+  bool reinitialize_position_on_first
+) {
+  GumboStringBuffer* buffer = &parser->_tokenizer_state->_tag_state._buffer;
+  if (buffer->length == 0 && reinitialize_position_on_first) {
+    reset_tag_buffer_start_point(parser);
+  }
+  gumbo_string_buffer_append_codepoint(codepoint, buffer);
+}
+
+// Like above but append a string.
+static void append_string_to_tag_buffer (
+  GumboParser* parser,
+  GumboStringPiece* str,
+  bool reinitialize_position_on_first
+) {
+  GumboStringBuffer* buffer = &parser->_tokenizer_state->_tag_state._buffer;
+  if (buffer->length == 0 && reinitialize_position_on_first) {
+    reset_tag_buffer_start_point(parser);
+  }
+  gumbo_string_buffer_append_string(str, buffer);
+}
+
+// (Re-)initialize the tag buffer. This also resets the original_text pointer
+// and _start_pos field to point to the current position.
+static void initialize_tag_buffer(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+
+  gumbo_string_buffer_init(&tag_state->_buffer);
+  reset_tag_buffer_start_point(parser);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#charref-in-attribute
+static bool character_reference_part_of_attribute(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  switch (tokenizer->_return_state) {
+  case GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED:
+  case GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED:
+  case GUMBO_LEX_ATTR_VALUE_UNQUOTED:
+    return true;
+  default:
+    return false;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#flush-code-points-consumed-as-a-character-reference
+// For each code point in the temporary buffer, add to the current attribute
+// value if the character reference was consumed as part of an attribute or
+// emit the code point as a character token.
+static StateResult flush_code_points_consumed_as_character_reference (
+  GumboParser* parser,
+  GumboToken* output
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  if (character_reference_part_of_attribute(parser)) {
+    const char *start = utf8iterator_get_mark_pointer(&tokenizer->_input);
+    assert(start);
+    GumboStringPiece str = {
+      .data = start,
+      .length = utf8iterator_get_char_pointer(&tokenizer->_input) - start,
+    };
+    bool unquoted = tokenizer->_return_state == GUMBO_LEX_ATTR_VALUE_UNQUOTED;
+    append_string_to_tag_buffer(parser, &str, unquoted);
+    return CONTINUE;
+  }
+  return emit_from_mark(parser, output);
+}
+
+// After a character reference has been successfully constructed, the standard
+// says to set the temporary buffer equal to the empty string, append the code
+// point(s) associated with the reference and flush code points consumed as a
+// character reference.
+// https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
+// That doesn't work for us because we use the temporary buffer in lock step
+// with the input for position and that would fail if we inserted a different
+// number of code points. So duplicate a bit of the above logic.
+static StateResult flush_char_ref (
+  GumboParser* parser,
+  int first,
+  int second,
+  GumboToken* output
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  if (character_reference_part_of_attribute(parser)) {
+    bool unquoted = tokenizer->_return_state == GUMBO_LEX_ATTR_VALUE_UNQUOTED;
+    append_char_to_tag_buffer(parser, first, unquoted);
+    if (second != kGumboNoChar)
+      append_char_to_tag_buffer(parser, second, unquoted);
+    return CONTINUE;
+  }
+  tokenizer->_buffered_emit_char = second;
+  return emit_char(parser, first, output);
+}
+
+
+// Initializes the tag_state to start a new tag, keeping track of the opening
+// positions and original text. Takes a boolean indicating whether this is a
+// start or end tag.
+static void start_new_tag(GumboParser* parser, bool is_start_tag) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+  int c = utf8iterator_current(&tokenizer->_input);
+  assert(is_alpha(c));
+  c = ensure_lowercase(c);
+  assert(is_alpha(c));
+
+  initialize_tag_buffer(parser);
+
+  assert(tag_state->_name == NULL);
+  assert(tag_state->_attributes.data == NULL);
+  // Initial size chosen by statistical analysis of a corpus of 60k webpages.
+  // 99.5% of elements have 0 attributes, 93% of the remainder have 1. These
+  // numbers are a bit higher for more modern websites (eg. ~45% = 0, ~40% = 1
+  // for the HTML5 Spec), but still have basically 99% of nodes with <= 2 attrs.
+  gumbo_vector_init(1, &tag_state->_attributes);
+  tag_state->_drop_next_attr_value = false;
+  tag_state->_is_start_tag = is_start_tag;
+  tag_state->_is_self_closing = false;
+  gumbo_debug("Starting new tag.\n");
+}
+
+// Fills in the specified char* with the contents of the tag buffer.
+static void copy_over_tag_buffer(GumboParser* parser, const char** output) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+  *output = gumbo_string_buffer_to_string(&tag_state->_buffer);
+}
+
+// Fills in:
+// * The original_text GumboStringPiece with the portion of the original
+// buffer that corresponds to the tag buffer.
+// * The start_pos GumboSourcePosition with the start position of the tag
+// buffer.
+// * The end_pos GumboSourcePosition with the current source position.
+static void copy_over_original_tag_text (
+  GumboParser* parser,
+  GumboStringPiece* original_text,
+  GumboSourcePosition* start_pos,
+  GumboSourcePosition* end_pos
+) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+
+  original_text->data = tag_state->_original_text;
+  original_text->length = utf8iterator_get_char_pointer(&tokenizer->_input) -
+                          tag_state->_original_text;
+  if (
+    original_text->length
+    && original_text->data[original_text->length - 1] == '\r'
+  ) {
+    // Since \r is skipped by the UTF-8 iterator, it can sometimes end up
+    // appended to the end of original text even when it's really the first part
+    // of the next character. If we detect this situation, shrink the length of
+    // the original text by 1 to remove the carriage return.
+    --original_text->length;
+  }
+  *start_pos = tag_state->_start_pos;
+  utf8iterator_get_position(&tokenizer->_input, end_pos);
+}
+
+// Releases and then re-initializes the tag buffer.
+static void reinitialize_tag_buffer(GumboParser* parser) {
+  gumbo_free(parser->_tokenizer_state->_tag_state._buffer.data);
+  initialize_tag_buffer(parser);
+}
+
+// Moves some data from the temporary buffer over the the tag-based fields in
+// TagState.
+static void finish_tag_name(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+
+  const char *data = tag_state->_buffer.data;
+  size_t length = tag_state->_buffer.length;
+  tag_state->_tag = gumbo_tagn_enum(data, length);
+  if (tag_state->_tag == GUMBO_TAG_UNKNOWN) {
+    char *name = gumbo_alloc(length + 1);
+    memcpy(name, data, length);
+    name[length] = 0;
+    tag_state->_name = name;
+  }
+  reinitialize_tag_buffer(parser);
+}
+
+// Adds an ERR_DUPLICATE_ATTR parse error to the parser's error struct.
+static void add_duplicate_attr_error(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboError* error = gumbo_add_error(parser);
+  if (!error) {
+    return;
+  }
+  GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
+  error->type = GUMBO_ERR_DUPLICATE_ATTRIBUTE;
+  error->position = tag_state->_start_pos;
+  error->original_text.data = tag_state->_original_text;
+  error->original_text.length =
+    utf8iterator_get_char_pointer(&tokenizer->_input) - error->original_text.data;
+  error->v.tokenizer.state = tokenizer->_state;
+}
+
+// Creates a new attribute in the current tag, copying the current tag buffer to
+// the attribute's name. The attribute's value starts out as the empty string
+// (following the "Boolean attributes" section of the spec) and is only
+// overwritten on finish_attribute_value(). If the attribute has already been
+// specified, the new attribute is dropped and a parse error is added
+static void finish_attribute_name(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  GumboTagState* tag_state = &tokenizer->_tag_state;
+  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
+
+  int max_attributes = parser->_options->max_attributes;
+  if (unlikely(max_attributes >= 0 && attributes->length >= (unsigned int) max_attributes)) {
+    parser->_output->status = GUMBO_STATUS_TOO_MANY_ATTRIBUTES;
+    gumbo_debug("Attributes limit exceeded.\n");
+    reinitialize_tag_buffer(parser);
+    tag_state->_drop_next_attr_value = true;
+    return;
+  }
+
+  // May've been set by a previous attribute without a value; reset it here.
+  tag_state->_drop_next_attr_value = false;
+  assert(tag_state->_attributes.data);
+  assert(tag_state->_attributes.capacity);
+
+  for (unsigned int i = 0; i < attributes->length; ++i) {
+    GumboAttribute* attr = attributes->data[i];
+    if (
+      strlen(attr->name) == tag_state->_buffer.length
+      && 0 == memcmp (
+        attr->name,
+        tag_state->_buffer.data,
+        tag_state->_buffer.length
+      )
+    ) {
+      // Identical attribute; bail.
+      add_duplicate_attr_error(parser);
+      reinitialize_tag_buffer(parser);
+      tag_state->_drop_next_attr_value = true;
+      return;
+    }
+  }
+
+  GumboAttribute* attr = gumbo_alloc(sizeof(GumboAttribute));
+  attr->attr_namespace = GUMBO_ATTR_NAMESPACE_NONE;
+  copy_over_tag_buffer(parser, &attr->name);
+  copy_over_original_tag_text (
+    parser,
+    &attr->original_name,
+    &attr->name_start,
+    &attr->name_end
+  );
+  attr->value = gumbo_strdup("");
+  copy_over_original_tag_text (
+    parser,
+    &attr->original_value,
+    &attr->name_start,
+    &attr->name_end
+  );
+  gumbo_vector_add(attr, attributes);
+  reinitialize_tag_buffer(parser);
+}
+
+// Finishes an attribute value. This sets the value of the most recently added
+// attribute to the current contents of the tag buffer.
+static void finish_attribute_value(GumboParser* parser) {
+  GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
+  if (tag_state->_drop_next_attr_value) {
+    // Duplicate attribute name detected in an earlier state, so we have to
+    // ignore the value.
+    tag_state->_drop_next_attr_value = false;
+    reinitialize_tag_buffer(parser);
+    return;
+  }
+
+  GumboAttribute* attr =
+      tag_state->_attributes.data[tag_state->_attributes.length - 1];
+  gumbo_free((void*) attr->value);
+  copy_over_tag_buffer(parser, &attr->value);
+  copy_over_original_tag_text(
+      parser, &attr->original_value, &attr->value_start, &attr->value_end);
+  reinitialize_tag_buffer(parser);
+}
+
+// Returns true if the current end tag matches the last start tag emitted.
+static bool is_appropriate_end_tag(GumboParser* parser) {
+  GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
+  assert(!tag_state->_is_start_tag);
+  return tag_state->_last_start_tag != GUMBO_TAG_LAST &&
+         tag_state->_last_start_tag == gumbo_tagn_enum(tag_state->_buffer.data,
+                                           tag_state->_buffer.length);
+}
+
+void gumbo_tokenizer_state_init (
+  GumboParser* parser,
+  const char* text,
+  size_t text_length
+) {
+  GumboTokenizerState* tokenizer = gumbo_alloc(sizeof(GumboTokenizerState));
+  parser->_tokenizer_state = tokenizer;
+  gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+  tokenizer->_return_state = GUMBO_LEX_DATA;
+  tokenizer->_character_reference_code = 0;
+  tokenizer->_reconsume_current_input = false;
+  tokenizer->_is_adjusted_current_node_foreign = false;
+  tokenizer->_is_in_cdata = false;
+  tokenizer->_tag_state._last_start_tag = GUMBO_TAG_LAST;
+  tokenizer->_tag_state._name = NULL;
+
+  tokenizer->_buffered_emit_char = kGumboNoChar;
+  gumbo_string_buffer_init(&tokenizer->_temporary_buffer);
+  tokenizer->_resume_pos = NULL;
+
+  mark_tag_state_as_empty(&tokenizer->_tag_state);
+
+  utf8iterator_init(parser, text, text_length, &tokenizer->_input);
+  utf8iterator_get_position(&tokenizer->_input, &tokenizer->_token_start_pos);
+  tokenizer->_token_start = utf8iterator_get_char_pointer(&tokenizer->_input);
+  doc_type_state_init(parser);
+}
+
+void gumbo_tokenizer_state_destroy(GumboParser* parser) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  assert(tokenizer->_doc_type_state.name == NULL);
+  assert(tokenizer->_doc_type_state.public_identifier == NULL);
+  assert(tokenizer->_doc_type_state.system_identifier == NULL);
+  gumbo_string_buffer_destroy(&tokenizer->_temporary_buffer);
+  assert(tokenizer->_tag_state._name == NULL);
+  assert(tokenizer->_tag_state._attributes.data == NULL);
+  gumbo_free(tokenizer);
+}
+
+void gumbo_tokenizer_set_state(GumboParser* parser, GumboTokenizerEnum state) {
+  parser->_tokenizer_state->_state = state;
+}
+
+void gumbo_tokenizer_set_is_adjusted_current_node_foreign (
+  GumboParser* parser,
+  bool is_foreign
+) {
+  if (is_foreign != parser->_tokenizer_state->_is_adjusted_current_node_foreign) {
+    gumbo_debug (
+      "Toggling is_current_node_foreign to %s.\n",
+      is_foreign ? "true" : "false"
+    );
+  }
+  parser->_tokenizer_state->_is_adjusted_current_node_foreign = is_foreign;
+}
+
+static void reconsume_in_state(GumboParser* parser, GumboTokenizerEnum state) {
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+  tokenizer->_reconsume_current_input = true;
+  tokenizer->_state = state;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#data-state
+static StateResult handle_data_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '&':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_DATA;
+      return CONTINUE;
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_TAG_OPEN);
+      set_mark(parser);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      return emit_char(parser, c, output);
+    case -1:
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-state
+static StateResult handle_rcdata_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '&':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_RCDATA;
+      return CONTINUE;
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA_LT);
+      set_mark(parser);
+      return CONTINUE;
+    case '\0':
+      return emit_replacement_char(parser, output);
+    case -1:
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-state
+static StateResult handle_rawtext_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT_LT);
+      set_mark(parser);
+      return CONTINUE;
+    case '\0':
+      return emit_replacement_char(parser, output);
+    case -1:
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-state
+static StateResult handle_script_data_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_LT);
+      set_mark(parser);
+      return CONTINUE;
+    case '\0':
+      return emit_replacement_char(parser, output);
+    case -1:
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#plaintext-state
+static StateResult handle_plaintext_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\0':
+      return emit_replacement_char(parser, output);
+    case -1:
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
+static StateResult handle_tag_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '!':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_MARKUP_DECLARATION_OPEN);
+      clear_temporary_buffer(parser);
+      return CONTINUE;
+    case '/':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_END_TAG_OPEN);
+      return CONTINUE;
+    case '?':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME);
+      clear_temporary_buffer(parser);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_BEFORE_TAG_NAME);
+      // Switch to data to emit EOF.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_from_mark(parser, output);
+    default:
+      if (is_alpha(c)) {
+        reconsume_in_state(parser, GUMBO_LEX_TAG_NAME);
+        start_new_tag(parser, true);
+        return CONTINUE;
+      }
+      tokenizer_add_parse_error(parser, GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
+static StateResult handle_end_tag_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_END_TAG_NAME);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_BEFORE_TAG_NAME);
+      // Similar to the tag open state except we need to emit '<' and '/'
+      // before the EOF.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_from_mark(parser, output);
+    default:
+      if (is_alpha(c)) {
+        reconsume_in_state(parser, GUMBO_LEX_TAG_NAME);
+        start_new_tag(parser, false);
+      } else {
+        tokenizer_add_parse_error(parser, GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME);
+        reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+        clear_temporary_buffer(parser);
+      }
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
+static StateResult handle_tag_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    case '/':
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    case '>':
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    default:
+      append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-less-than-sign-state
+static StateResult handle_rcdata_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (c == '/') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RCDATA_END_TAG_OPEN);
+    return CONTINUE;
+  } else {
+    reconsume_in_state(parser, GUMBO_LEX_RCDATA);
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-open-state
+static StateResult handle_rcdata_end_tag_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_RCDATA_END_TAG_NAME);
+    start_new_tag(parser, false);
+    return CONTINUE;
+  }
+  reconsume_in_state(parser, GUMBO_LEX_RCDATA);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-name-state
+static StateResult handle_rcdata_end_tag_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  UNUSED_IF_NDEBUG(tokenizer);
+  if (is_alpha(c)) {
+    append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    }
+    break;
+  }
+  abandon_current_tag(parser);
+  reconsume_in_state(parser, GUMBO_LEX_RCDATA);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-less-than-sign-state
+static StateResult handle_rawtext_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (c == '/') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_RAWTEXT_END_TAG_OPEN);
+    return CONTINUE;
+  } else {
+    reconsume_in_state(parser, GUMBO_LEX_RAWTEXT);
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-open-state
+static StateResult handle_rawtext_end_tag_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_RAWTEXT_END_TAG_NAME);
+    start_new_tag(parser, false);
+    return CONTINUE;
+  } else {
+    reconsume_in_state(parser, GUMBO_LEX_RAWTEXT);
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-name-state
+static StateResult handle_rawtext_end_tag_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    }
+    break;
+  }
+  abandon_current_tag(parser);
+  reconsume_in_state(parser, GUMBO_LEX_RAWTEXT);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-less-than-sign-state
+static StateResult handle_script_data_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (c == '/') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_END_TAG_OPEN);
+    return CONTINUE;
+  }
+  if (c == '!') {
+    // This is the only place we don't reconsume the input before emitting the
+    // temporary buffer. Since the current position is stored and the current
+    // character is not emitted, we need to advance the input and then
+    // reconsume.
+    utf8iterator_next(&tokenizer->_input);
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_START);
+    return emit_from_mark(parser, output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-open-state
+static StateResult handle_script_data_end_tag_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_END_TAG_NAME);
+    start_new_tag(parser, false);
+    return CONTINUE;
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-name-state
+static StateResult handle_script_data_end_tag_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    }
+    break;
+  }
+  abandon_current_tag(parser);
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-state
+static StateResult handle_script_data_escaped_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (c == '-') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_START_DASH);
+    return emit_char(parser, c, output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-dash-state
+static StateResult handle_script_data_escaped_start_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (c == '-') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH);
+    return emit_char(parser, c, output);
+  } else {
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-state
+static StateResult handle_script_data_escaped_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH);
+      return emit_char(parser, c, output);
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT);
+      clear_temporary_buffer(parser);
+      set_mark(parser);
+      return CONTINUE;
+    case '\0':
+      return emit_replacement_char(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-state
+static StateResult handle_script_data_escaped_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH);
+      return emit_char(parser, c, output);
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT);
+      clear_temporary_buffer(parser);
+      set_mark(parser);
+      return CONTINUE;
+    case '\0':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+      return emit_replacement_char(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
+    default:
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-dash-state
+static StateResult handle_script_data_escaped_dash_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      return emit_char(parser, c, output);
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT);
+      clear_temporary_buffer(parser);
+      set_mark(parser);
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA);
+      return emit_char(parser, c, output);
+    case '\0':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+      return emit_replacement_char(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
+    default:
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-less-than-sign-state
+static StateResult handle_script_data_escaped_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  assert(temporary_buffer_is_empty(parser));
+  if (c == '/') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_OPEN);
+    return CONTINUE;
+  }
+  if (is_alpha(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_START);
+    return emit_from_mark(parser, output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-open-state
+static StateResult handle_script_data_escaped_end_tag_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_NAME);
+    start_new_tag(parser, false);
+    return CONTINUE;
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-name-state
+static StateResult handle_script_data_escaped_end_tag_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (is_alpha(c)) {
+    append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
+    return CONTINUE;
+  }
+  switch (c) {
+  case '\t':
+  case '\n':
+  case '\f':
+  case ' ':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    }
+    break;
+  case '/':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    }
+    break;
+  case '>':
+    if (is_appropriate_end_tag(parser)) {
+      finish_tag_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    }
+    break;
+  }
+  abandon_current_tag(parser);
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return emit_from_mark(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-start-state
+static StateResult handle_script_data_double_escaped_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+    case '/':
+    case '>':
+      gumbo_tokenizer_set_state (
+        parser,
+        gumbo_string_equals (
+          &kScriptTag,
+          (GumboStringPiece*) &tokenizer->_temporary_buffer
+        )
+        ? GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED
+        : GUMBO_LEX_SCRIPT_DATA_ESCAPED
+      );
+      return emit_char(parser, c, output);
+  }
+  if (is_alpha(c)) {
+    append_char_to_temporary_buffer(parser, ensure_lowercase(c));
+    return emit_char(parser, c, output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_ESCAPED);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-state
+static StateResult handle_script_data_double_escaped_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH);
+      return emit_char(parser, c, output);
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT);
+      return emit_char(parser, c, output);
+    case '\0':
+      return emit_replacement_char(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
+    default:
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-state
+static StateResult handle_script_data_double_escaped_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH);
+      return emit_char(parser, c, output);
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT);
+      return emit_char(parser, c, output);
+    case '\0':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_replacement_char(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
+    default:
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-dash-state
+static StateResult handle_script_data_double_escaped_dash_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      return emit_char(parser, c, output);
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT);
+      return emit_char(parser, c, output);
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA);
+      return emit_char(parser, c, output);
+    case '\0':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_replacement_char(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT);
+      return emit_eof(parser, output);
+    default:
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-less-than-sign-state
+static StateResult handle_script_data_double_escaped_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (c == '/') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_END);
+    clear_temporary_buffer(parser);
+    return emit_char(parser, c, output);
+  } else {
+    reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-end-state
+static StateResult handle_script_data_double_escaped_end_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+    case '/':
+    case '>':
+      gumbo_tokenizer_set_state(
+          parser, gumbo_string_equals(&kScriptTag,
+                      (GumboStringPiece*) &tokenizer->_temporary_buffer)
+                      ? GUMBO_LEX_SCRIPT_DATA_ESCAPED
+                      : GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+      return emit_char(parser, c, output);
+  }
+  if (is_alpha(c)) {
+    append_char_to_temporary_buffer(parser, ensure_lowercase(c));
+    return emit_char(parser, c, output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state
+static StateResult handle_before_attr_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '/':
+    case '>':
+    case -1:
+      reconsume_in_state(parser, GUMBO_LEX_AFTER_ATTR_NAME);
+      return CONTINUE;
+    case '=':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_NAME);
+      append_char_to_tag_buffer(parser, c, true);
+      return CONTINUE;
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_ATTR_NAME);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
+static StateResult handle_attr_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+    case '/':
+    case '>':
+    case -1:
+      finish_attribute_name(parser);
+      reconsume_in_state(parser, GUMBO_LEX_AFTER_ATTR_NAME);
+      return CONTINUE;
+    case '=':
+      finish_attribute_name(parser);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_VALUE);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
+      return CONTINUE;
+    case '"':
+    case '\'':
+    case '<':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME);
+    // Fall through.
+    default:
+      append_char_to_tag_buffer(parser, ensure_lowercase(c), true);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-name-state
+static StateResult handle_after_attr_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '/':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    case '=':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_VALUE);
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_ATTR_NAME);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-value-state
+static StateResult handle_before_attr_value_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '"':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED);
+      reset_tag_buffer_start_point(parser);
+      return CONTINUE;
+    case '\'':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED);
+      reset_tag_buffer_start_point(parser);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_ATTRIBUTE_VALUE);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_ATTR_VALUE_UNQUOTED);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-double-quoted-state
+static StateResult handle_attr_value_double_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '"':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED);
+      return CONTINUE;
+    case '&':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED;
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_tag_buffer(parser, kUtf8ReplacementChar, false);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    default:
+      append_char_to_tag_buffer(parser, c, false);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-single-quoted-state
+static StateResult handle_attr_value_single_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\'':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED);
+      return CONTINUE;
+    case '&':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED;
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_tag_buffer(parser, kUtf8ReplacementChar, false);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    default:
+      append_char_to_tag_buffer(parser, c, false);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-unquoted-state
+static StateResult handle_attr_value_unquoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      finish_attribute_value(parser);
+      return CONTINUE;
+    case '&':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_CHARACTER_REFERENCE);
+      set_mark(parser);
+      tokenizer->_return_state = GUMBO_LEX_ATTR_VALUE_UNQUOTED;
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      finish_attribute_value(parser);
+      return emit_current_tag(parser, output);
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_tag_buffer(parser, kUtf8ReplacementChar, true);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    case '"':
+    case '\'':
+    case '<':
+    case '=':
+    case '`':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE);
+    // Fall through.
+    default:
+      append_char_to_tag_buffer(parser, c, true);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-value-quoted-state
+static StateResult handle_after_attr_value_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  finish_attribute_value(parser);
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+    case '/':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_SELF_CLOSING_START_TAG);
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_current_tag(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES);
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#self-closing-start-tag-state
+static StateResult handle_self_closing_start_tag_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_tag_state._is_self_closing = true;
+      return emit_current_tag(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_TAG);
+      abandon_current_tag(parser);
+      return emit_eof(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG);
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_ATTR_NAME);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
+static StateResult handle_bogus_comment_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '>':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+    return emit_comment(parser, output);
+  case -1:
+    // We need to emit the comment and then the EOF, so reconsume in data
+    // state.
+    reconsume_in_state(parser, GUMBO_LEX_DATA);
+    return emit_comment(parser, output);
+  case '\0':
+    tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+    append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+    return CONTINUE;
+  default:
+    append_char_to_temporary_buffer(parser, c);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
+static StateResult handle_markup_declaration_open_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int UNUSED_ARG(c),
+  GumboToken* UNUSED_ARG(output)
+) {
+  if (
+    utf8iterator_maybe_consume_match (
+      &tokenizer->_input,
+      "--",
+      sizeof("--") - 1,
+      /* case sensitive */ true
+    )
+  ) {
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_START);
+    return CONTINUE;
+  }
+  if (
+    utf8iterator_maybe_consume_match (
+      &tokenizer->_input,
+      "DOCTYPE",
+      sizeof("DOCTYPE") - 1,
+      /* case sensitive */ false
+    )
+  ) {
+    reconsume_in_state(parser, GUMBO_LEX_DOCTYPE);
+    // If we get here, we know we'll eventually emit a doctype token, so now is
+    // the time to initialize the doctype strings. (Not in doctype_state_init,
+    // since then they'll leak if ownership never gets transferred to the
+    // doctype token.
+    tokenizer->_doc_type_state.name = gumbo_strdup("");
+    tokenizer->_doc_type_state.public_identifier = gumbo_strdup("");
+    tokenizer->_doc_type_state.system_identifier = gumbo_strdup("");
+    return CONTINUE;
+  }
+  if (
+    utf8iterator_maybe_consume_match (
+      &tokenizer->_input,
+      "[CDATA[", sizeof("[CDATA[") - 1,
+      /* case sensitive */ true
+    )
+  ) {
+    if (tokenizer->_is_adjusted_current_node_foreign) {
+      reconsume_in_state(parser, GUMBO_LEX_CDATA_SECTION);
+      tokenizer->_is_in_cdata = true;
+      // Start the token after the <![CDATA[.
+      reset_token_start_point(tokenizer);
+    } else {
+      tokenizer_add_token_parse_error(parser, GUMBO_ERR_CDATA_IN_HTML_CONTENT);
+      clear_temporary_buffer(parser);
+      append_string_to_temporary_buffer (
+        parser,
+        &(const GumboStringPiece) { .data = "[CDATA[", .length = 7 }
+      );
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+    }
+    return CONTINUE;
+  }
+  tokenizer_add_parse_error(parser, GUMBO_ERR_INCORRECTLY_OPENED_COMMENT);
+  reconsume_in_state(parser, GUMBO_LEX_BOGUS_COMMENT);
+  clear_temporary_buffer(parser);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
+static StateResult handle_comment_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_START_DASH);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-start-dash-state
+static StateResult handle_comment_start_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit the EOF next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+      append_char_to_temporary_buffer(parser, '-');
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-state
+static StateResult handle_comment_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '<':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT);
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END_DASH);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit the EOF token next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    default:
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-state
+static StateResult handle_comment_lt_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '!':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT_BANG);
+    append_char_to_temporary_buffer(parser, c);
+    return CONTINUE;
+  case '<':
+    append_char_to_temporary_buffer(parser, c);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-state
+static StateResult handle_comment_lt_bang_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '-':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT_BANG_DASH);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-state
+static StateResult handle_comment_lt_bang_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '-':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_LT_BANG_DASH_DASH);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_END_DASH);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-dash-state
+static StateResult handle_comment_lt_bang_dash_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '>':
+  case -1:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_END);
+    return CONTINUE;
+  default:
+    tokenizer_add_parse_error(parser, GUMBO_ERR_NESTED_COMMENT);
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT_END);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-end-dash-state
+static StateResult handle_comment_end_dash_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '-':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END);
+    return CONTINUE;
+  case -1:
+    tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+    // Switch to data to emit EOF next.
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+    return emit_comment(parser, output);
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+    append_char_to_temporary_buffer(parser, '-');
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-end-state
+static StateResult handle_comment_end_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    case '!':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END_BANG);
+      return CONTINUE;
+    case '-':
+      append_char_to_temporary_buffer(parser, '-');
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit EOF next.
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+      append_char_to_temporary_buffer(parser, '-');
+      append_char_to_temporary_buffer(parser, '-');
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#comment-end-bang-state
+static StateResult handle_comment_end_bang_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '-':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_COMMENT_END_DASH);
+      append_char_to_temporary_buffer(parser, '-');
+      append_char_to_temporary_buffer(parser, '-');
+      append_char_to_temporary_buffer(parser, '!');
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_COMMENT);
+      // Switch to data to emit EOF next.
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_comment(parser, output);
+    default:
+      reconsume_in_state(parser, GUMBO_LEX_COMMENT);
+      append_char_to_temporary_buffer(parser, '-');
+      append_char_to_temporary_buffer(parser, '-');
+      append_char_to_temporary_buffer(parser, '!');
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#doctype-state
+static StateResult handle_doctype_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  assert(temporary_buffer_is_empty(parser));
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
+      return CONTINUE;
+    case '>':
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      // Switch to data to emit EOF next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME);
+      reconsume_in_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_NAME);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state
+static StateResult handle_before_doctype_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DOCTYPE_NAME);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_NAME);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      // Switch to data to emit EOF next.
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
+    default:
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DOCTYPE_NAME);
+      append_char_to_temporary_buffer(parser, ensure_lowercase(c));
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state
+static StateResult handle_doctype_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_NAME);
+      gumbo_free((void*) tokenizer->_doc_type_state.name);
+      finish_temporary_buffer(parser, &tokenizer->_doc_type_state.name);
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      gumbo_free((void*) tokenizer->_doc_type_state.name);
+      finish_temporary_buffer(parser, &tokenizer->_doc_type_state.name);
+      return emit_doctype(parser, output);
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      gumbo_free((void*) tokenizer->_doc_type_state.name);
+      finish_temporary_buffer(parser, &tokenizer->_doc_type_state.name);
+      return emit_doctype(parser, output);
+    default:
+      append_char_to_temporary_buffer(parser, ensure_lowercase(c));
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state
+static StateResult handle_after_doctype_name_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      if (utf8iterator_maybe_consume_match(
+              &tokenizer->_input, "PUBLIC", sizeof("PUBLIC") - 1, false)) {
+        reconsume_in_state(parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD);
+      } else if (utf8iterator_maybe_consume_match(&tokenizer->_input, "SYSTEM",
+                     sizeof("SYSTEM") - 1, false)) {
+        reconsume_in_state(parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD);
+      } else {
+        tokenizer_add_parse_error(
+            parser, GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME);
+        reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+        tokenizer->_doc_type_state.force_quirks = true;
+      }
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-keyword-state
+static StateResult handle_after_doctype_public_keyword_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID);
+      return CONTINUE;
+    case '"':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD);
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED);
+      return CONTINUE;
+    case '\'':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD);
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-public-identifier-state
+static StateResult handle_before_doctype_public_id_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '"':
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED);
+      return CONTINUE;
+    case '\'':
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(double-quoted)-state
+static StateResult handle_doctype_public_id_double_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '"':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID);
+      finish_doctype_public_id(parser);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_public_id(parser);
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_public_id(parser);
+      return emit_doctype(parser, output);
+    default:
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(single-quoted)-state
+static StateResult handle_doctype_public_id_single_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\'':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID);
+      finish_doctype_public_id(parser);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_public_id(parser);
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_public_id(parser);
+      return emit_doctype(parser, output);
+    default:
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-identifier-state
+static StateResult handle_after_doctype_public_id_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID);
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
+    case '"':
+      tokenizer_add_parse_error (
+        parser,
+        GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS
+      );
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
+      return CONTINUE;
+    case '\'':
+      tokenizer_add_parse_error (
+        parser,
+        GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS
+      );
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#between-doctype-public-and-system-identifiers-state
+static StateResult handle_between_doctype_public_system_id_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
+    case '"':
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
+      return CONTINUE;
+    case '\'':
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
+      return CONTINUE;
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-keyword-state
+static StateResult handle_after_doctype_system_keyword_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID);
+      return CONTINUE;
+    case '"':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD);
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
+      return CONTINUE;
+    case '\'':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD);
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-system-identifier-state
+static StateResult handle_before_doctype_system_id_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '"':
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED);
+      return CONTINUE;
+    case '\'':
+      assert(temporary_buffer_is_empty(parser));
+      gumbo_tokenizer_set_state(
+          parser, GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(double-quoted)-state
+static StateResult handle_doctype_system_id_double_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '"':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID);
+      finish_doctype_system_id(parser);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_system_id(parser);
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_system_id(parser);
+      return emit_doctype(parser, output);
+    default:
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(single-quoted)-state
+static StateResult handle_doctype_system_id_single_quoted_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\'':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID);
+      finish_doctype_system_id(parser);
+      return CONTINUE;
+    case '\0':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+      append_char_to_temporary_buffer(parser, kUtf8ReplacementChar);
+      return CONTINUE;
+    case '>':
+      tokenizer_add_parse_error(parser, GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER);
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_system_id(parser);
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      finish_doctype_system_id(parser);
+      return emit_doctype(parser, output);
+    default:
+      append_char_to_temporary_buffer(parser, c);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-identifier-state
+static StateResult handle_after_doctype_system_id_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+    case '\t':
+    case '\n':
+    case '\f':
+    case ' ':
+      return CONTINUE;
+    case '>':
+      gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+      return emit_doctype(parser, output);
+    case -1:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_DOCTYPE);
+      reconsume_in_state(parser, GUMBO_LEX_DATA);
+      tokenizer->_doc_type_state.force_quirks = true;
+      return emit_doctype(parser, output);
+    default:
+      tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER);
+      reconsume_in_state(parser, GUMBO_LEX_BOGUS_DOCTYPE);
+      return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#bogus-doctype-state
+static StateResult handle_bogus_doctype_state (
+  GumboParser* parser,
+  GumboTokenizerState* UNUSED_ARG(tokenizer),
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case '>':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
+    return emit_doctype(parser, output);
+  case '\0':
+    tokenizer_add_parse_error(parser, GUMBO_ERR_UNEXPECTED_NULL_CHARACTER);
+    return CONTINUE;
+  case -1:
+    reconsume_in_state(parser, GUMBO_LEX_DATA);
+    return emit_doctype(parser, output);
+  default:
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-state
+static StateResult handle_cdata_section_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case ']':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA_SECTION_BRACKET);
+    set_mark(parser);
+    return CONTINUE;
+  case -1:
+    tokenizer_add_parse_error(parser, GUMBO_ERR_EOF_IN_CDATA);
+    return emit_eof(parser, output);
+  default:
+    return emit_char(parser, c, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-bracket-state
+static StateResult handle_cdata_section_bracket_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case ']':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA_SECTION_END);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_CDATA_SECTION);
+    // Emit the ].
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-end-state
+static StateResult handle_cdata_section_end_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  switch (c) {
+  case ']':
+  {
+    // XXX: This is terrible. We want to emit a ] corresponding to the first
+    // of the three in a row we've seen. So let's emit one token from the
+    // temporary buffer (which will rewind 3 characters, emit the ] and
+    // advance one). Next, let's clear the temporary buffer which will set the
+    // mark to the middle of the three brackets. Finally, let's move to the
+    // appropriate state.
+    StateResult result = emit_from_mark(parser, output);
+    tokenizer->_resume_pos = NULL;
+    set_mark(parser);
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_CDATA_SECTION);
+    return result;
+  }
+  case '>':
+    // We're done with CDATA so move past the >, reset the token start point
+    // to point after the >, and then reconsume in the data state.
+    utf8iterator_next(&tokenizer->_input);
+    reset_token_start_point(tokenizer);
+    reconsume_in_state(parser, GUMBO_LEX_DATA);
+    tokenizer->_is_in_cdata = false;
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_CDATA_SECTION);
+    return emit_from_mark(parser, output);
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state
+static StateResult handle_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isalnum(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_NAMED_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  if (c == '#') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+static StateResult handle_named_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  const char *cur = utf8iterator_get_char_pointer(&tokenizer->_input);
+  const char *end = utf8iterator_get_end_pointer(&tokenizer->_input);
+  int code_point[2];
+  size_t size = match_named_char_ref(cur, end - cur, code_point);
+
+  if (size > 0) {
+    utf8iterator_maybe_consume_match(&tokenizer->_input, cur, size, true);
+    int next = utf8iterator_current(&tokenizer->_input);
+    reconsume_in_state(parser, tokenizer->_return_state);
+    if (character_reference_part_of_attribute(parser)
+        && cur[size-1] != ';'
+        && (next == '=' || gumbo_ascii_isalnum(next))) {
+      GumboStringPiece str = { .data = cur, .length = size };
+      append_string_to_temporary_buffer(parser, &str);
+      return flush_code_points_consumed_as_character_reference(parser, output);
+    }
+    if (cur[size-1] != ';')
+      tokenizer_add_char_ref_error(parser, GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE, -1);
+    reconsume_in_state(parser, tokenizer->_return_state);
+    return flush_char_ref(parser, code_point[0], code_point[1], output);
+  }
+  reconsume_in_state(parser, GUMBO_LEX_AMBIGUOUS_AMPERSAND);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#ambiguous-ampersand-state
+static StateResult handle_ambiguous_ampersand_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isalnum(c)) {
+    if (character_reference_part_of_attribute(parser)) {
+      append_char_to_tag_buffer(parser, c, true);
+      return CONTINUE;
+    }
+    return emit_char(parser, c, output);
+  }
+  if (c == ';') {
+      tokenizer_add_char_ref_error(parser, GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE, -1);
+    // fall through
+  }
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-state
+static StateResult handle_numeric_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  tokenizer->_character_reference_code = 0;
+  switch (c) {
+  case 'x':
+  case 'X':
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START);
+    return CONTINUE;
+  default:
+    reconsume_in_state(parser, GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE_START);
+    return CONTINUE;
+  }
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#hexademical-character-reference-start-state
+static StateResult handle_hexadecimal_character_reference_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isxdigit(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error (
+    parser,
+    GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE,
+    -1
+  );
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state
+static StateResult handle_decimal_character_reference_start_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isdigit(c)) {
+    reconsume_in_state(parser, GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error (
+    parser,
+    GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE,
+    -1
+  );
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_code_points_consumed_as_character_reference(parser, output);
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#hexademical-character-reference-state
+static StateResult handle_hexadecimal_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 16 + (c - 0x0030);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (gumbo_ascii_isupper_xdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 16 + (c - 0x0037);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (gumbo_ascii_islower_xdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 16 + (c - 0x0057);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (c == ';') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error(
+    parser,
+    GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE,
+    tokenizer->_character_reference_code
+  );
+  reconsume_in_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-state
+static StateResult handle_decimal_character_reference_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  if (gumbo_ascii_isdigit(c)) {
+    tokenizer->_character_reference_code =
+      tokenizer->_character_reference_code * 10 + (c - 0x0030);
+    if (tokenizer->_character_reference_code > kUtf8MaxChar)
+      tokenizer->_character_reference_code = kUtf8MaxChar+1;
+    return CONTINUE;
+  }
+  if (c == ';') {
+    gumbo_tokenizer_set_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+    return CONTINUE;
+  }
+  tokenizer_add_char_ref_error(
+    parser,
+    GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE,
+    tokenizer->_character_reference_code
+  );
+  reconsume_in_state(parser, GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END);
+  return CONTINUE;
+}
+
+// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
+static StateResult handle_numeric_character_reference_end_state (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+) {
+  c = tokenizer->_character_reference_code;
+  if (c == 0) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_NULL_CHARACTER_REFERENCE,
+      c
+    );
+    c = kUtf8ReplacementChar;
+  } else if (c > kUtf8MaxChar) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE,
+      c
+    );
+    c = kUtf8ReplacementChar;
+  } else if (utf8_is_surrogate(c)) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE,
+      c
+    );
+    c = kUtf8ReplacementChar;
+  } else if (utf8_is_noncharacter(c)) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE,
+      c
+    );
+  } else if (c == 0x0D || (utf8_is_control(c) && !gumbo_ascii_isspace(c))) {
+    tokenizer_add_char_ref_error(
+      parser,
+      GUMBO_ERR_CONTROL_CHARACTER_REFERENCE,
+      c
+    );
+    switch (c) {
+    case 0x80: c = 0x20AC; break;
+    case 0x82: c = 0x201A; break;
+    case 0x83: c = 0x0192; break;
+    case 0x84: c = 0x201E; break;
+    case 0x85: c = 0x2026; break;
+    case 0x86: c = 0x2020; break;
+    case 0x87: c = 0x2021; break;
+    case 0x88: c = 0x02C6; break;
+    case 0x89: c = 0x2030; break;
+    case 0x8A: c = 0x0160; break;
+    case 0x8B: c = 0x2039; break;
+    case 0x8C: c = 0x0152; break;
+    case 0x8E: c = 0x017D; break;
+    case 0x91: c = 0x2018; break;
+    case 0x92: c = 0x2019; break;
+    case 0x93: c = 0x201C; break;
+    case 0x94: c = 0x201D; break;
+    case 0x95: c = 0x2022; break;
+    case 0x96: c = 0x2013; break;
+    case 0x97: c = 0x2014; break;
+    case 0x98: c = 0x02DC; break;
+    case 0x99: c = 0x2122; break;
+    case 0x9A: c = 0x0161; break;
+    case 0x9B: c = 0x203A; break;
+    case 0x9C: c = 0x0153; break;
+    case 0x9E: c = 0x017E; break;
+    case 0x9F: c = 0x0178; break;
+    }
+  }
+  reconsume_in_state(parser, tokenizer->_return_state);
+  return flush_char_ref(parser, c, kGumboNoChar, output);
+}
+
+typedef StateResult (*GumboLexerStateFunction) (
+  GumboParser* parser,
+  GumboTokenizerState* tokenizer,
+  int c,
+  GumboToken* output
+);
+
+static GumboLexerStateFunction dispatch_table[] = {
+  [GUMBO_LEX_DATA] = handle_data_state,
+  [GUMBO_LEX_RCDATA] = handle_rcdata_state,
+  [GUMBO_LEX_RAWTEXT] = handle_rawtext_state,
+  [GUMBO_LEX_SCRIPT_DATA] = handle_script_data_state,
+  [GUMBO_LEX_PLAINTEXT] = handle_plaintext_state,
+  [GUMBO_LEX_TAG_OPEN] = handle_tag_open_state,
+  [GUMBO_LEX_END_TAG_OPEN] = handle_end_tag_open_state,
+  [GUMBO_LEX_TAG_NAME] = handle_tag_name_state,
+  [GUMBO_LEX_RCDATA_LT] = handle_rcdata_lt_state,
+  [GUMBO_LEX_RCDATA_END_TAG_OPEN] = handle_rcdata_end_tag_open_state,
+  [GUMBO_LEX_RCDATA_END_TAG_NAME] = handle_rcdata_end_tag_name_state,
+  [GUMBO_LEX_RAWTEXT_LT] = handle_rawtext_lt_state,
+  [GUMBO_LEX_RAWTEXT_END_TAG_OPEN] = handle_rawtext_end_tag_open_state,
+  [GUMBO_LEX_RAWTEXT_END_TAG_NAME] = handle_rawtext_end_tag_name_state,
+  [GUMBO_LEX_SCRIPT_DATA_LT] = handle_script_data_lt_state,
+  [GUMBO_LEX_SCRIPT_DATA_END_TAG_OPEN] = handle_script_data_end_tag_open_state,
+  [GUMBO_LEX_SCRIPT_DATA_END_TAG_NAME] = handle_script_data_end_tag_name_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_START] = handle_script_data_escaped_start_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_START_DASH] = handle_script_data_escaped_start_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED] = handle_script_data_escaped_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH] = handle_script_data_escaped_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH] = handle_script_data_escaped_dash_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT] = handle_script_data_escaped_lt_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_OPEN] = handle_script_data_escaped_end_tag_open_state,
+  [GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_NAME] = handle_script_data_escaped_end_tag_name_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_START] = handle_script_data_double_escaped_start_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED] = handle_script_data_double_escaped_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH] = handle_script_data_double_escaped_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH] = handle_script_data_double_escaped_dash_dash_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT] = handle_script_data_double_escaped_lt_state,
+  [GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_END] = handle_script_data_double_escaped_end_state,
+  [GUMBO_LEX_BEFORE_ATTR_NAME] = handle_before_attr_name_state,
+  [GUMBO_LEX_ATTR_NAME] = handle_attr_name_state,
+  [GUMBO_LEX_AFTER_ATTR_NAME] = handle_after_attr_name_state,
+  [GUMBO_LEX_BEFORE_ATTR_VALUE] = handle_before_attr_value_state,
+  [GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED] = handle_attr_value_double_quoted_state,
+  [GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED] = handle_attr_value_single_quoted_state,
+  [GUMBO_LEX_ATTR_VALUE_UNQUOTED] = handle_attr_value_unquoted_state,
+  [GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED] = handle_after_attr_value_quoted_state,
+  [GUMBO_LEX_SELF_CLOSING_START_TAG] = handle_self_closing_start_tag_state,
+  [GUMBO_LEX_BOGUS_COMMENT] = handle_bogus_comment_state,
+  [GUMBO_LEX_MARKUP_DECLARATION_OPEN] = handle_markup_declaration_open_state,
+  [GUMBO_LEX_COMMENT_START] = handle_comment_start_state,
+  [GUMBO_LEX_COMMENT_START_DASH] = handle_comment_start_dash_state,
+  [GUMBO_LEX_COMMENT] = handle_comment_state,
+  [GUMBO_LEX_COMMENT_LT] = handle_comment_lt_state,
+  [GUMBO_LEX_COMMENT_LT_BANG] = handle_comment_lt_bang_state,
+  [GUMBO_LEX_COMMENT_LT_BANG_DASH] = handle_comment_lt_bang_dash_state,
+  [GUMBO_LEX_COMMENT_LT_BANG_DASH_DASH] = handle_comment_lt_bang_dash_dash_state,
+  [GUMBO_LEX_COMMENT_END_DASH] = handle_comment_end_dash_state,
+  [GUMBO_LEX_COMMENT_END] = handle_comment_end_state,
+  [GUMBO_LEX_COMMENT_END_BANG] = handle_comment_end_bang_state,
+  [GUMBO_LEX_DOCTYPE] = handle_doctype_state,
+  [GUMBO_LEX_BEFORE_DOCTYPE_NAME] = handle_before_doctype_name_state,
+  [GUMBO_LEX_DOCTYPE_NAME] = handle_doctype_name_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_NAME] = handle_after_doctype_name_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD] = handle_after_doctype_public_keyword_state,
+  [GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID] = handle_before_doctype_public_id_state,
+  [GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED] = handle_doctype_public_id_double_quoted_state,
+  [GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED] = handle_doctype_public_id_single_quoted_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID] = handle_after_doctype_public_id_state,
+  [GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID] = handle_between_doctype_public_system_id_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD] = handle_after_doctype_system_keyword_state,
+  [GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID] = handle_before_doctype_system_id_state,
+  [GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED] = handle_doctype_system_id_double_quoted_state,
+  [GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED] = handle_doctype_system_id_single_quoted_state,
+  [GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID] = handle_after_doctype_system_id_state,
+  [GUMBO_LEX_BOGUS_DOCTYPE] = handle_bogus_doctype_state,
+  [GUMBO_LEX_CDATA_SECTION] = handle_cdata_section_state,
+  [GUMBO_LEX_CDATA_SECTION_BRACKET] = handle_cdata_section_bracket_state,
+  [GUMBO_LEX_CDATA_SECTION_END] = handle_cdata_section_end_state,
+  [GUMBO_LEX_CHARACTER_REFERENCE] = handle_character_reference_state,
+  [GUMBO_LEX_NAMED_CHARACTER_REFERENCE] = handle_named_character_reference_state,
+  [GUMBO_LEX_AMBIGUOUS_AMPERSAND] = handle_ambiguous_ampersand_state,
+  [GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE] = handle_numeric_character_reference_state,
+  [GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START] = handle_hexadecimal_character_reference_start_state,
+  [GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE_START] = handle_decimal_character_reference_start_state,
+  [GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE] = handle_hexadecimal_character_reference_state,
+  [GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE] = handle_decimal_character_reference_state,
+  [GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END] = handle_numeric_character_reference_end_state,
+};
+
+void gumbo_lex(GumboParser* parser, GumboToken* output) {
+  // Because of the spec requirements that...
+  //
+  // 1. Tokens be handled immediately by the parser upon emission.
+  // 2. Some states (eg. CDATA, or various error conditions) require the
+  // emission of multiple tokens in the same states.
+  // 3. The tokenizer often has to reconsume the same character in a different
+  // state.
+  //
+  // ...all state must be held in the GumboTokenizer struct instead of in local
+  // variables in this function. That allows us to return from this method with
+  // a token, and then immediately jump back to the same state with the same
+  // input if we need to return a different token. The various emit_* functions
+  // are responsible for changing state (eg. flushing the chardata buffer,
+  // reading the next input character) to avoid an infinite loop.
+  GumboTokenizerState* tokenizer = parser->_tokenizer_state;
+
+  if (tokenizer->_buffered_emit_char != kGumboNoChar) {
+    tokenizer->_reconsume_current_input = true;
+    emit_char(parser, tokenizer->_buffered_emit_char, output);
+    // And now that we've avoided advancing the input, make sure we set
+    // _reconsume_current_input back to false to make sure the *next* character
+    // isn't consumed twice.
+    tokenizer->_reconsume_current_input = false;
+    tokenizer->_buffered_emit_char = kGumboNoChar;
+    return;
+  }
+
+  if (maybe_emit_from_mark(parser, output) == EMIT_TOKEN) {
+    return;
+  }
+
+  while (1) {
+    assert(!tokenizer->_resume_pos);
+    assert(tokenizer->_buffered_emit_char == kGumboNoChar);
+    int c = utf8iterator_current(&tokenizer->_input);
+    GumboTokenizerEnum state = tokenizer->_state;
+    gumbo_debug("Lexing character '%c' (%d) in state %u.\n", c, c, state);
+    StateResult result = dispatch_table[state](parser, tokenizer, c, output);
+    // We need to clear reconsume_current_input before returning to prevent
+    // certain infinite loop states.
+    bool should_advance = !tokenizer->_reconsume_current_input;
+    tokenizer->_reconsume_current_input = false;
+
+    if (result == EMIT_TOKEN)
+      return;
+
+    if (should_advance) {
+      utf8iterator_next(&tokenizer->_input);
+    }
+  }
+}
+
+void gumbo_token_destroy(GumboToken* token) {
+  if (!token) return;
+
+  switch (token->type) {
+    case GUMBO_TOKEN_DOCTYPE:
+      gumbo_free((void*) token->v.doc_type.name);
+      gumbo_free((void*) token->v.doc_type.public_identifier);
+      gumbo_free((void*) token->v.doc_type.system_identifier);
+      return;
+    case GUMBO_TOKEN_START_TAG:
+      for (unsigned int i = 0; i < token->v.start_tag.attributes.length; ++i) {
+        GumboAttribute* attr = token->v.start_tag.attributes.data[i];
+        if (attr) {
+          // May have been nulled out if this token was merged with another.
+          gumbo_destroy_attribute(attr);
+        }
+      }
+      gumbo_free((void*) token->v.start_tag.attributes.data);
+      if (token->v.start_tag.tag == GUMBO_TAG_UNKNOWN) {
+        gumbo_free(token->v.start_tag.name);
+        token->v.start_tag.name = NULL;
+      }
+      return;
+    case GUMBO_TOKEN_END_TAG:
+      if (token->v.end_tag.tag == GUMBO_TAG_UNKNOWN) {
+        gumbo_free(token->v.end_tag.name);
+        token->v.end_tag.name = NULL;
+      }
+      break;
+    case GUMBO_TOKEN_COMMENT:
+      gumbo_free((void*) token->v.text);
+      return;
+    default:
+      return;
+  }
+}