nokogumbo 1.5.0 → 2.0.3

data/gumbo-parser/src/tokenizer.h

@@ -1,25 +1,9 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains an implementation of a tokenizer for HTML5.  It consumes a
-// buffer of UTF-8 characters, and then emits a stream of tokens.
-
 #ifndef GUMBO_TOKENIZER_H_
 #define GUMBO_TOKENIZER_H_
 
+// This contains an implementation of a tokenizer for HTML5. It consumes a
+// buffer of UTF-8 characters, and then emits a stream of tokens.
+
 #include <stdbool.h>
 #include <stddef.h>
 
@@ -49,11 +33,20 @@ typedef struct GumboInternalTokenDocType {
 // Struct containing all information pertaining to start tag tokens.
 typedef struct GumboInternalTokenStartTag {
   GumboTag tag;
+  // NULL unless tag is GUMBO_TAG_UNKNOWN
+  char *name;
   GumboVector /* GumboAttribute */ attributes;
   bool is_self_closing;
 } GumboTokenStartTag;
 
-// A data structure representing a single token in the input stream.  This
+// Struct containing all information pertaining to end tag tokens.
+typedef struct GumboInternalTokenEndTag {
+  GumboTag tag;
+  // NULL unless tag is GUMBO_TAG_UNKNOWN
+  char *name;
+} GumboTokenEndTag;
+
+// A data structure representing a single token in the input stream. This
 // contains an enum for the type, the source position, a GumboStringPiece
 // pointing to the original text, and then a union for any parsed data.
 typedef struct GumboInternalToken {
@@ -63,7 +56,7 @@ typedef struct GumboInternalToken {
   union {
     GumboTokenDocType doc_type;
     GumboTokenStartTag start_tag;
-    GumboTag end_tag;
+    GumboTokenEndTag end_tag;
     const char* text;  // For comments.
     int character;     // For character, whitespace, null, and EOF tokens.
   } v;
@@ -71,53 +64,49 @@ typedef struct GumboInternalToken {
 
 // Initializes the tokenizer state within the GumboParser object, setting up a
 // parse of the specified text.
-void gumbo_tokenizer_state_init(
-    struct GumboInternalParser* parser, const char* text, size_t text_length);
+void gumbo_tokenizer_state_init (
+  struct GumboInternalParser* parser,
+  const char* text,
+  size_t text_length
+);
 
 // Destroys the tokenizer state within the GumboParser object, freeing any
 // dynamically-allocated structures within it.
 void gumbo_tokenizer_state_destroy(struct GumboInternalParser* parser);
 
-// Sets the tokenizer state to the specified value.  This is needed by some
+// Sets the tokenizer state to the specified value. This is needed by some
 // parser states, which alter the state of the tokenizer in response to tags
 // seen.
-void gumbo_tokenizer_set_state(
-    struct GumboInternalParser* parser, GumboTokenizerEnum state);
-
-// Flags whether the current node is a foreign content element.  This is
-// necessary for the markup declaration open state, where the tokenizer must be
-// aware of the state of the parser to properly tokenize bad comment tags.
-// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#markup-declaration-open-state
-void gumbo_tokenizer_set_is_current_node_foreign(
-    struct GumboInternalParser* parser, bool is_foreign);
+void gumbo_tokenizer_set_state (
+  struct GumboInternalParser* parser,
+  GumboTokenizerEnum state
+);
+
+// Flags whether the adjusted current node is a foreign content element. This
+// is necessary for the markup declaration open state, where the tokenizer
+// must be aware of the state of the parser to properly tokenize bad comment
+// tags.
+// https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
+void gumbo_tokenizer_set_is_adjusted_current_node_foreign (
+  struct GumboInternalParser* parser,
+  bool is_foreign
+);
 
 // Lexes a single token from the specified buffer, filling the output with the
-// parsed GumboToken data structure.  Returns true for a successful
-// tokenization, false if a parse error occurs.
-//
-// Example:
-//   struct GumboInternalParser parser;
-//   GumboToken output;
-//   gumbo_tokenizer_state_init(&parser, text, strlen(text));
-//   while (gumbo_lex(&parser, &output)) {
-//     ...do stuff with output.
-//     gumbo_token_destroy(&parser, &token);
-//   }
-//   gumbo_tokenizer_state_destroy(&parser);
-bool gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
-
-// Frees the internally-allocated pointers within an GumboToken.  Note that this
+// parsed GumboToken data structure.
+void gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
+
+// Frees the internally-allocated pointers within a GumboToken. Note that this
 // doesn't free the token itself, since oftentimes it will be allocated on the
-// stack.  A simple call to free() (or GumboParser->deallocator, if
-// appropriate) can handle that.
+// stack.
 //
 // Note that if you are handing over ownership of the internal strings to some
 // other data structure - for example, a parse tree - these do not need to be
 // freed.
-void gumbo_token_destroy(struct GumboInternalParser* parser, GumboToken* token);
+void gumbo_token_destroy(GumboToken* token);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // GUMBO_TOKENIZER_H_
+#endif // GUMBO_TOKENIZER_H_

data/gumbo-parser/src/tokenizer_states.h

@@ -1,103 +1,339 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains the list of states used in the tokenizer.  Although at first
+#ifndef GUMBO_TOKENIZER_STATES_H_
+#define GUMBO_TOKENIZER_STATES_H_
+
+// This contains the list of states used in the tokenizer. Although at first
 // glance it seems like these could be kept internal to the tokenizer, several
 // of the actions in the parser require that it reach into the tokenizer and
-// reset the tokenizer state.  For that to work, it needs to have the
+// reset the tokenizer state. For that to work, it needs to have the
 // definitions of individual states available.
 //
 // This may also be useful for providing more detailed error messages for parse
 // errors, as we can match up states and inputs in a table without having to
 // clutter the tokenizer code with lots of precise error messages.
 
-#ifndef GUMBO_TOKENIZER_STATES_H_
-#define GUMBO_TOKENIZER_STATES_H_
-
 // The ordering of this enum is also used to build the dispatch table for the
 // tokenizer state machine, so if it is changed, be sure to update that too.
 typedef enum {
+  // 12.2.5.1 Data state
+  // https://html.spec.whatwg.org/multipage/parsing.html#data-state
   GUMBO_LEX_DATA,
-  GUMBO_LEX_CHAR_REF_IN_DATA,
+
+  // 12.2.5.2 RCDATA state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-state
   GUMBO_LEX_RCDATA,
-  GUMBO_LEX_CHAR_REF_IN_RCDATA,
+
+  // 12.2.5.3 RAWTEXT state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-state<Paste>
   GUMBO_LEX_RAWTEXT,
-  GUMBO_LEX_SCRIPT,
+
+  // 12.2.5.4 Script data state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-state
+  GUMBO_LEX_SCRIPT_DATA,
+
+  // 12.2.5.5 PLAINTEXT state
+  // https://html.spec.whatwg.org/multipage/parsing.html#plaintext-state
   GUMBO_LEX_PLAINTEXT,
+
+  // 12.2.5.6 Tag open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
   GUMBO_LEX_TAG_OPEN,
+
+  // 12.2.5.7 End tag open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
   GUMBO_LEX_END_TAG_OPEN,
+
+  // 12.2.5.8 Tag name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
   GUMBO_LEX_TAG_NAME,
+
+  // 12.2.5.9 RCDATA less-than sign state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-less-than-sign-state
   GUMBO_LEX_RCDATA_LT,
+
+  // 12.2.5.10 RCDATA end tag open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-open-state
   GUMBO_LEX_RCDATA_END_TAG_OPEN,
+
+  // 12.2.5.11 RCDATA end tag name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-name-state
   GUMBO_LEX_RCDATA_END_TAG_NAME,
+
+  // 12.2.5.12 RAWTEXT less-than sign state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-less-than-sign-state
   GUMBO_LEX_RAWTEXT_LT,
+
+  // 12.2.5.13 RAWTEXT end tag open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-open-state
   GUMBO_LEX_RAWTEXT_END_TAG_OPEN,
+
+  // 12.2.5.14 RAWTEXT end tag name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-name-state
   GUMBO_LEX_RAWTEXT_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_LT,
-  GUMBO_LEX_SCRIPT_END_TAG_OPEN,
-  GUMBO_LEX_SCRIPT_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_ESCAPED_START,
-  GUMBO_LEX_SCRIPT_ESCAPED_START_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED,
-  GUMBO_LEX_SCRIPT_ESCAPED_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED_LT,
-  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN,
-  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END,
+
+  // 12.2.5.15 Script data less-than sign state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-less-than-sign-state
+  GUMBO_LEX_SCRIPT_DATA_LT,
+
+  // 12.2.5.16 Script data end tag open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-open-state
+  GUMBO_LEX_SCRIPT_DATA_END_TAG_OPEN,
+
+  // 12.2.5.17 Script data end tag name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-name-state
+  GUMBO_LEX_SCRIPT_DATA_END_TAG_NAME,
+
+  // 12.2.5.18 Script data escape start state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_START,
+
+  // 12.2.5.19 Script data escape start dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-dash-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_START_DASH,
+
+  // 12.2.5.20 Script data escaped state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED,
+
+  // 12.2.5.21 Script data escaped dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH,
+
+  // 12.2.5.22 Script data escaped dash dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-dash-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH,
+
+  // 12.2.5.23 Script data escaped less than sign state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-less-than-sign-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT,
+
+  // 12.2.5.24 Script data escaped end tag open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-open-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_OPEN,
+
+  // 12.2.5.25 Script data escaped end tag name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-name-state
+  GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_NAME,
+
+  // 12.2.5.26 Script data double escape start state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-start-state
+  GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_START,
+
+  // 12.2.5.27 Script data double escaped state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-state
+  GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED,
+
+  // 12.2.5.28 Script data double escaped dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-state
+  GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH,
+
+  // 12.2.5.29 Script data double escaped dash dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-dash-state
+  GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH,
+
+  // 12.2.5.30 Script data double escaped less-than sign state
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-less-than-sign-state
+  GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT,
+
+  // 12.2.5.31 Script data double escape end state (XXX: spec bug with the
+  // name?)
+  // https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-end-state
+  GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_END,
+
+  // 12.2.5.32 Before attribute name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state
   GUMBO_LEX_BEFORE_ATTR_NAME,
+
+  // 12.2.5.33 Attributet name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
   GUMBO_LEX_ATTR_NAME,
+
+  // 12.2.5.34 After attribute name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-name-state
   GUMBO_LEX_AFTER_ATTR_NAME,
+
+  // 12.2.5.35 Before attribute value state
+  // https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-value-state
   GUMBO_LEX_BEFORE_ATTR_VALUE,
+
+  // 12.2.5.36 Attribute value (double-quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(double-quoted)-state
   GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED,
+
+  // 12.2.5.37 Attribute value (single-quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(single-quoted)-state
   GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED,
+
+  // 12.2.5.38 Attribute value (unquoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(unquoted)-state
   GUMBO_LEX_ATTR_VALUE_UNQUOTED,
-  GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE,
+
+  // 12.2.5.39 After attribute value (quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-value-(quoted)-state
   GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED,
+
+  // 12.2.5.40 Self-closing start tag state
+  // https://html.spec.whatwg.org/multipage/parsing.html#self-closing-start-tag-state
   GUMBO_LEX_SELF_CLOSING_START_TAG,
+
+  // 12.2.5.41 Bogus comment state
+  // https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
   GUMBO_LEX_BOGUS_COMMENT,
-  GUMBO_LEX_MARKUP_DECLARATION,
+
+  // 12.2.5.42 Markup declaration open state
+  // https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
+  GUMBO_LEX_MARKUP_DECLARATION_OPEN,
+
+  // 12.2.5.43 Comment start state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
   GUMBO_LEX_COMMENT_START,
+
+  // 12.2.5.44 Comment start dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-start-dash-state
   GUMBO_LEX_COMMENT_START_DASH,
+
+  // 12.2.5.45 Comment state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-state
   GUMBO_LEX_COMMENT,
+
+  // 12.2.5.46 Comment less-than sign state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-state
+  GUMBO_LEX_COMMENT_LT,
+
+  // 12.2.5.47 Comment less-than sign bang state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-state
+  GUMBO_LEX_COMMENT_LT_BANG,
+
+  // 12.2.5.48 Comment less-than sign bang dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-state
+  GUMBO_LEX_COMMENT_LT_BANG_DASH,
+
+  // 12.2.5.49 Comment less-than sign bang dash dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-dash-state
+  GUMBO_LEX_COMMENT_LT_BANG_DASH_DASH,
+
+  // 12.2.5.50 Comment end dash state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-end-dash-state
   GUMBO_LEX_COMMENT_END_DASH,
+
+  // 12.2.5.51 Comment end state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-end-state
   GUMBO_LEX_COMMENT_END,
+
+  // 12.2.5.52 Comment end bang state
+  // https://html.spec.whatwg.org/multipage/parsing.html#comment-end-bang-state
   GUMBO_LEX_COMMENT_END_BANG,
+
+  // 12.2.5.53 DOCTYPE state
+  // https://html.spec.whatwg.org/multipage/parsing.html#doctype-state
   GUMBO_LEX_DOCTYPE,
+
+  // 12.2.5.54 Before DOCTYPE name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state
   GUMBO_LEX_BEFORE_DOCTYPE_NAME,
+
+  // 12.2.5.55 DOCTYPE name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state
   GUMBO_LEX_DOCTYPE_NAME,
+
+  // 12.2.5.56 After DOCTYPE name state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state
   GUMBO_LEX_AFTER_DOCTYPE_NAME,
+
+  // 12.2.5.57 After DOCTYPE public keyword state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-keyword-state
   GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD,
+
+  // 12.2.5.58 Before DOCTYPE public identifier state
+  // https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-public-identifier-state
   GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID,
+
+  // 12.2.5.59 DOCTYPE public identifier (double-quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(double-quoted)-state
   GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED,
+
+  // 12.2.5.60 DOCTYPE public identifier (single-quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(single-quoted)-state
   GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED,
+
+  // 12.2.5.61 After DOCTYPE public identifier state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-identifier-state
   GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID,
+
+  // 12.2.5.62 Between DOCTYPE public and system identifiers state
+  // https://html.spec.whatwg.org/multipage/parsing.html#between-doctype-public-and-system-identifiers-state
   GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID,
+
+  // 12.2.5.63 After DOCTYPE system keyword state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-keyword-state
   GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD,
+
+  // 12.2.5.64 Before DOCTYPE system identifier state
+  // https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-system-identifier-state
   GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID,
+
+  // 12.2.5.65 DOCTYPE system identifier (double-quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(double-quoted)-state
   GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED,
+
+  // 12.2.5.66 DOCTYPE system identifier (single-quoted) state
+  // https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(single-quoted)-state
   GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED,
+
+  // 12.2.5.67 After DOCTYPE system identifier state
+  // https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-identifier-state
   GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID,
+
+  // 12.2.5.68 Bogus DOCTYPE state
+  // https://html.spec.whatwg.org/multipage/parsing.html#bogus-doctype-state
   GUMBO_LEX_BOGUS_DOCTYPE,
-  GUMBO_LEX_CDATA
+
+  // 12.2.5.69 CDATA section state
+  // https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-state
+  GUMBO_LEX_CDATA_SECTION,
+
+  // 12.2.5.70 CDATA section bracket state
+  // https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-bracket-state
+  GUMBO_LEX_CDATA_SECTION_BRACKET,
+
+  // 12.2.5.71 CDATA section end state
+  // https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-end-state
+  GUMBO_LEX_CDATA_SECTION_END,
+
+  // 12.2.5.72 Character reference state
+  // https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state
+  GUMBO_LEX_CHARACTER_REFERENCE,
+
+  // 12.2.5.73 Named character reference state
+  // https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
+  GUMBO_LEX_NAMED_CHARACTER_REFERENCE,
+
+  // 12.2.5.74 Ambiguous ampersand state
+  // https://html.spec.whatwg.org/multipage/parsing.html#ambiguous-ampersand-state
+  GUMBO_LEX_AMBIGUOUS_AMPERSAND,
+
+  // 12.2.5.75 Numeric character reference state
+  // https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-state
+  GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE,
+
+  // 12.2.5.76 Hexadecimal character reference start state
+  // https://html.spec.whatwg.org/multipage/parsing.html#hexadecimal-character-reference-start-state
+  GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START,
+
+  // 12.2.5.77 Decimal character reference start state
+  // https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state
+  GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE_START,
+  
+  // 12.2.5.78 Hexadecimal character reference state
+  // https://html.spec.whatwg.org/multipage/parsing.html#hexadecimal-character-reference-state
+  GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE,
+  
+  // 12.2.5.79 Decimal character reference state
+  // https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-state
+  GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE,
+
+  // 12.2.5.80 Numeric character reference end state
+  // https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
+  GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END
 } GumboTokenizerEnum;
 
-#endif  // GUMBO_TOKENIZER_STATES_H_
+#endif // GUMBO_TOKENIZER_STATES_H_