nokogumbo 1.4.8 → 1.4.9

data/gumbo-parser/src/tokenizer.h

@@ -0,0 +1,123 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// This contains an implementation of a tokenizer for HTML5.  It consumes a
+// buffer of UTF-8 characters, and then emits a stream of tokens.
+
+#ifndef GUMBO_TOKENIZER_H_
+#define GUMBO_TOKENIZER_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "gumbo.h"
+#include "token_type.h"
+#include "tokenizer_states.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct GumboInternalParser;
+
+// Struct containing all information pertaining to doctype tokens.
+typedef struct GumboInternalTokenDocType {
+  const char* name;
+  const char* public_identifier;
+  const char* system_identifier;
+  bool force_quirks;
+  // There's no way to tell a 0-length public or system ID apart from the
+  // absence of a public or system ID, but they're handled different by the
+  // spec, so we need bool flags for them.
+  bool has_public_identifier;
+  bool has_system_identifier;
+} GumboTokenDocType;
+
+// Struct containing all information pertaining to start tag tokens.
+typedef struct GumboInternalTokenStartTag {
+  GumboTag tag;
+  GumboVector /* GumboAttribute */ attributes;
+  bool is_self_closing;
+} GumboTokenStartTag;
+
+// A data structure representing a single token in the input stream.  This
+// contains an enum for the type, the source position, a GumboStringPiece
+// pointing to the original text, and then a union for any parsed data.
+typedef struct GumboInternalToken {
+  GumboTokenType type;
+  GumboSourcePosition position;
+  GumboStringPiece original_text;
+  union {
+    GumboTokenDocType doc_type;
+    GumboTokenStartTag start_tag;
+    GumboTag end_tag;
+    const char* text;  // For comments.
+    int character;     // For character, whitespace, null, and EOF tokens.
+  } v;
+} GumboToken;
+
+// Initializes the tokenizer state within the GumboParser object, setting up a
+// parse of the specified text.
+void gumbo_tokenizer_state_init(
+    struct GumboInternalParser* parser, const char* text, size_t text_length);
+
+// Destroys the tokenizer state within the GumboParser object, freeing any
+// dynamically-allocated structures within it.
+void gumbo_tokenizer_state_destroy(struct GumboInternalParser* parser);
+
+// Sets the tokenizer state to the specified value.  This is needed by some
+// parser states, which alter the state of the tokenizer in response to tags
+// seen.
+void gumbo_tokenizer_set_state(
+    struct GumboInternalParser* parser, GumboTokenizerEnum state);
+
+// Flags whether the current node is a foreign content element.  This is
+// necessary for the markup declaration open state, where the tokenizer must be
+// aware of the state of the parser to properly tokenize bad comment tags.
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#markup-declaration-open-state
+void gumbo_tokenizer_set_is_current_node_foreign(
+    struct GumboInternalParser* parser, bool is_foreign);
+
+// Lexes a single token from the specified buffer, filling the output with the
+// parsed GumboToken data structure.  Returns true for a successful
+// tokenization, false if a parse error occurs.
+//
+// Example:
+//   struct GumboInternalParser parser;
+//   GumboToken output;
+//   gumbo_tokenizer_state_init(&parser, text, strlen(text));
+//   while (gumbo_lex(&parser, &output)) {
+//     ...do stuff with output.
+//     gumbo_token_destroy(&parser, &token);
+//   }
+//   gumbo_tokenizer_state_destroy(&parser);
+bool gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
+
+// Frees the internally-allocated pointers within an GumboToken.  Note that this
+// doesn't free the token itself, since oftentimes it will be allocated on the
+// stack.  A simple call to free() (or GumboParser->deallocator, if
+// appropriate) can handle that.
+//
+// Note that if you are handing over ownership of the internal strings to some
+// other data structure - for example, a parse tree - these do not need to be
+// freed.
+void gumbo_token_destroy(struct GumboInternalParser* parser, GumboToken* token);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_TOKENIZER_H_

data/gumbo-parser/src/tokenizer_states.h

@@ -0,0 +1,103 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// This contains the list of states used in the tokenizer.  Although at first
+// glance it seems like these could be kept internal to the tokenizer, several
+// of the actions in the parser require that it reach into the tokenizer and
+// reset the tokenizer state.  For that to work, it needs to have the
+// definitions of individual states available.
+//
+// This may also be useful for providing more detailed error messages for parse
+// errors, as we can match up states and inputs in a table without having to
+// clutter the tokenizer code with lots of precise error messages.
+
+#ifndef GUMBO_TOKENIZER_STATES_H_
+#define GUMBO_TOKENIZER_STATES_H_
+
+// The ordering of this enum is also used to build the dispatch table for the
+// tokenizer state machine, so if it is changed, be sure to update that too.
+typedef enum {
+  GUMBO_LEX_DATA,
+  GUMBO_LEX_CHAR_REF_IN_DATA,
+  GUMBO_LEX_RCDATA,
+  GUMBO_LEX_CHAR_REF_IN_RCDATA,
+  GUMBO_LEX_RAWTEXT,
+  GUMBO_LEX_SCRIPT,
+  GUMBO_LEX_PLAINTEXT,
+  GUMBO_LEX_TAG_OPEN,
+  GUMBO_LEX_END_TAG_OPEN,
+  GUMBO_LEX_TAG_NAME,
+  GUMBO_LEX_RCDATA_LT,
+  GUMBO_LEX_RCDATA_END_TAG_OPEN,
+  GUMBO_LEX_RCDATA_END_TAG_NAME,
+  GUMBO_LEX_RAWTEXT_LT,
+  GUMBO_LEX_RAWTEXT_END_TAG_OPEN,
+  GUMBO_LEX_RAWTEXT_END_TAG_NAME,
+  GUMBO_LEX_SCRIPT_LT,
+  GUMBO_LEX_SCRIPT_END_TAG_OPEN,
+  GUMBO_LEX_SCRIPT_END_TAG_NAME,
+  GUMBO_LEX_SCRIPT_ESCAPED_START,
+  GUMBO_LEX_SCRIPT_ESCAPED_START_DASH,
+  GUMBO_LEX_SCRIPT_ESCAPED,
+  GUMBO_LEX_SCRIPT_ESCAPED_DASH,
+  GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH,
+  GUMBO_LEX_SCRIPT_ESCAPED_LT,
+  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN,
+  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME,
+  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START,
+  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED,
+  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH,
+  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH,
+  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT,
+  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END,
+  GUMBO_LEX_BEFORE_ATTR_NAME,
+  GUMBO_LEX_ATTR_NAME,
+  GUMBO_LEX_AFTER_ATTR_NAME,
+  GUMBO_LEX_BEFORE_ATTR_VALUE,
+  GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED,
+  GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED,
+  GUMBO_LEX_ATTR_VALUE_UNQUOTED,
+  GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE,
+  GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED,
+  GUMBO_LEX_SELF_CLOSING_START_TAG,
+  GUMBO_LEX_BOGUS_COMMENT,
+  GUMBO_LEX_MARKUP_DECLARATION,
+  GUMBO_LEX_COMMENT_START,
+  GUMBO_LEX_COMMENT_START_DASH,
+  GUMBO_LEX_COMMENT,
+  GUMBO_LEX_COMMENT_END_DASH,
+  GUMBO_LEX_COMMENT_END,
+  GUMBO_LEX_COMMENT_END_BANG,
+  GUMBO_LEX_DOCTYPE,
+  GUMBO_LEX_BEFORE_DOCTYPE_NAME,
+  GUMBO_LEX_DOCTYPE_NAME,
+  GUMBO_LEX_AFTER_DOCTYPE_NAME,
+  GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD,
+  GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID,
+  GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED,
+  GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED,
+  GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID,
+  GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID,
+  GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD,
+  GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID,
+  GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED,
+  GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED,
+  GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID,
+  GUMBO_LEX_BOGUS_DOCTYPE,
+  GUMBO_LEX_CDATA
+} GumboTokenizerEnum;
+
+#endif  // GUMBO_TOKENIZER_STATES_H_

data/gumbo-parser/src/utf8.c

@@ -0,0 +1,270 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "utf8.h"
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>  // For strncasecmp.
+
+#include "error.h"
+#include "gumbo.h"
+#include "parser.h"
+#include "util.h"
+#include "vector.h"
+
+const int kUtf8ReplacementChar = 0xFFFD;
+
+// Reference material:
+// Wikipedia: http://en.wikipedia.org/wiki/UTF-8#Description
+// RFC 3629: http://tools.ietf.org/html/rfc3629
+// HTML5 Unicode handling:
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html#preprocessing-the-input-stream
+//
+// This implementation is based on a DFA-based decoder by Bjoern Hoehrmann
+// <bjoern@hoehrmann.de>.  We wrap the inner table-based decoder routine in our
+// own handling for newlines, tabs, invalid continuation bytes, and other
+// conditions that the HTML5 spec fully specifies but normal UTF8 decoders do
+// not handle.
+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.  Full text of
+// the license agreement and code follows.
+
+// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to
+// do
+// so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+
+#define UTF8_ACCEPT 0
+#define UTF8_REJECT 12
+
+static const uint8_t utf8d[] = {
+    // The first part of the table maps bytes to character classes that
+    // to reduce the size of the transition table and create bitmasks.
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9,
+    9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 2, 2, 2, 2, 2, 2,
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10,
+    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8,
+    8, 8, 8, 8, 8, 8,
+
+    // The second part is a transition table that maps a combination
+    // of a state of the automaton and a character class to a state.
+    0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12,
+    12, 12, 12, 12, 12, 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12,
+    12, 12, 12, 12, 24, 12, 24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12,
+    12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12,
+    12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12,
+    12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
+};
+
+uint32_t static inline decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
+  uint32_t type = utf8d[byte];
+
+  *codep = (*state != UTF8_ACCEPT) ? (byte & 0x3fu) | (*codep << 6)
+                                   : (0xff >> type) & (byte);
+
+  *state = utf8d[256 + *state + type];
+  return *state;
+}
+
+// END COPIED CODE.
+
+// Adds a decoding error to the parser's error list, based on the current state
+// of the Utf8Iterator.
+static void add_error(Utf8Iterator* iter, GumboErrorType type) {
+  GumboParser* parser = iter->_parser;
+
+  GumboError* error = gumbo_add_error(parser);
+  if (!error) {
+    return;
+  }
+  error->type = type;
+  error->position = iter->_pos;
+  error->original_text = iter->_start;
+
+  // At the point the error is recorded, the code point hasn't been computed
+  // yet (and can't be, because it's invalid), so we need to build up the raw
+  // hex value from the bytes under the cursor.
+  uint64_t code_point = 0;
+  for (int i = 0; i < iter->_width; ++i) {
+    code_point = (code_point << 8) | (unsigned char) iter->_start[i];
+  }
+  error->v.codepoint = code_point;
+}
+
+// Reads the next UTF-8 character in the iter.
+// This assumes that iter->_start points to the beginning of the character.
+// When this method returns, iter->_width and iter->_current will be set
+// appropriately, as well as any error flags.
+static void read_char(Utf8Iterator* iter) {
+  if (iter->_start >= iter->_end) {
+    // No input left to consume; emit an EOF and set width = 0.
+    iter->_current = -1;
+    iter->_width = 0;
+    return;
+  }
+
+  uint32_t code_point = 0;
+  uint32_t state = UTF8_ACCEPT;
+  for (const char* c = iter->_start; c < iter->_end; ++c) {
+    decode(&state, &code_point, (uint32_t)(unsigned char) (*c));
+    if (state == UTF8_ACCEPT) {
+      iter->_width = c - iter->_start + 1;
+      // This is the special handling for carriage returns that is mandated by
+      // the HTML5 spec.  Since we're looking for particular 7-bit literal
+      // characters, we operate in terms of chars and only need a check for iter
+      // overrun, instead of having to read in a full next code point.
+      // http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#preprocessing-the-input-stream
+      if (code_point == '\r') {
+        assert(iter->_width == 1);
+        const char* next = c + 1;
+        if (next < iter->_end && *next == '\n') {
+          // Advance the iter, as if the carriage return didn't exist.
+          ++iter->_start;
+          // Preserve the true offset, since other tools that look at it may be
+          // unaware of HTML5's rules for converting \r into \n.
+          ++iter->_pos.offset;
+        }
+        code_point = '\n';
+      }
+      if (utf8_is_invalid_code_point(code_point)) {
+        add_error(iter, GUMBO_ERR_UTF8_INVALID);
+        code_point = kUtf8ReplacementChar;
+      }
+      iter->_current = code_point;
+      return;
+    } else if (state == UTF8_REJECT) {
+      // We don't want to consume the invalid continuation byte of a multi-byte
+      // run, but we do want to skip past an invalid first byte.
+      iter->_width = c - iter->_start + (c == iter->_start);
+      iter->_current = kUtf8ReplacementChar;
+      add_error(iter, GUMBO_ERR_UTF8_INVALID);
+      return;
+    }
+  }
+  // If we got here without exiting early, then we've reached the end of the
+  // iterator.  Add an error for truncated input, set the width to consume the
+  // rest of the iterator, and emit a replacement character.  The next time we
+  // enter this method, it will detect that there's no input to consume and
+  // output an EOF.
+  iter->_current = kUtf8ReplacementChar;
+  iter->_width = iter->_end - iter->_start;
+  add_error(iter, GUMBO_ERR_UTF8_TRUNCATED);
+}
+
+static void update_position(Utf8Iterator* iter) {
+  iter->_pos.offset += iter->_width;
+  if (iter->_current == '\n') {
+    ++iter->_pos.line;
+    iter->_pos.column = 1;
+  } else if (iter->_current == '\t') {
+    int tab_stop = iter->_parser->_options->tab_stop;
+    iter->_pos.column = ((iter->_pos.column / tab_stop) + 1) * tab_stop;
+  } else if (iter->_current != -1) {
+    ++iter->_pos.column;
+  }
+}
+
+// Returns true if this Unicode code point is in the list of characters
+// forbidden by the HTML5 spec, such as undefined control chars.
+bool utf8_is_invalid_code_point(int c) {
+  return (c >= 0x1 && c <= 0x8) || c == 0xB || (c >= 0xE && c <= 0x1F) ||
+         (c >= 0x7F && c <= 0x9F) || (c >= 0xFDD0 && c <= 0xFDEF) ||
+         ((c & 0xFFFF) == 0xFFFE) || ((c & 0xFFFF) == 0xFFFF);
+}
+
+void utf8iterator_init(GumboParser* parser, const char* source,
+    size_t source_length, Utf8Iterator* iter) {
+  iter->_start = source;
+  iter->_end = source + source_length;
+  iter->_pos.line = 1;
+  iter->_pos.column = 1;
+  iter->_pos.offset = 0;
+  iter->_parser = parser;
+  read_char(iter);
+}
+
+void utf8iterator_next(Utf8Iterator* iter) {
+  // We update positions based on the *last* character read, so that the first
+  // character following a newline is at column 1 in the next line.
+  update_position(iter);
+  iter->_start += iter->_width;
+  read_char(iter);
+}
+
+int utf8iterator_current(const Utf8Iterator* iter) { return iter->_current; }
+
+void utf8iterator_get_position(
+    const Utf8Iterator* iter, GumboSourcePosition* output) {
+  *output = iter->_pos;
+}
+
+const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter) {
+  return iter->_start;
+}
+
+const char* utf8iterator_get_end_pointer(const Utf8Iterator* iter) {
+  return iter->_end;
+}
+
+bool utf8iterator_maybe_consume_match(Utf8Iterator* iter, const char* prefix,
+    size_t length, bool case_sensitive) {
+  bool matched = (iter->_start + length <= iter->_end) &&
+                 (case_sensitive ? !strncmp(iter->_start, prefix, length)
+                                 : !strncasecmp(iter->_start, prefix, length));
+  if (matched) {
+    for (unsigned int i = 0; i < length; ++i) {
+      utf8iterator_next(iter);
+    }
+    return true;
+  } else {
+    return false;
+  }
+}
+
+void utf8iterator_mark(Utf8Iterator* iter) {
+  iter->_mark = iter->_start;
+  iter->_mark_pos = iter->_pos;
+}
+
+// Returns the current input stream position to the mark.
+void utf8iterator_reset(Utf8Iterator* iter) {
+  iter->_start = iter->_mark;
+  iter->_pos = iter->_mark_pos;
+  read_char(iter);
+}
+
+// Sets the position and original text fields of an error to the value at the
+// mark.
+void utf8iterator_fill_error_at_mark(Utf8Iterator* iter, GumboError* error) {
+  error->position = iter->_mark_pos;
+  error->original_text = iter->_mark;
+}