nokogumbo 1.4.7 → 1.4.8

data/gumbo-parser/src/tokenizer.h

@@ -1,123 +0,0 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains an implementation of a tokenizer for HTML5.  It consumes a
-// buffer of UTF-8 characters, and then emits a stream of tokens.
-
-#ifndef GUMBO_TOKENIZER_H_
-#define GUMBO_TOKENIZER_H_
-
-#include <stdbool.h>
-#include <stddef.h>
-
-#include "gumbo.h"
-#include "token_type.h"
-#include "tokenizer_states.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct GumboInternalParser;
-
-// Struct containing all information pertaining to doctype tokens.
-typedef struct GumboInternalTokenDocType {
-  const char* name;
-  const char* public_identifier;
-  const char* system_identifier;
-  bool force_quirks;
-  // There's no way to tell a 0-length public or system ID apart from the
-  // absence of a public or system ID, but they're handled different by the
-  // spec, so we need bool flags for them.
-  bool has_public_identifier;
-  bool has_system_identifier;
-} GumboTokenDocType;
-
-// Struct containing all information pertaining to start tag tokens.
-typedef struct GumboInternalTokenStartTag {
-  GumboTag tag;
-  GumboVector /* GumboAttribute */ attributes;
-  bool is_self_closing;
-} GumboTokenStartTag;
-
-// A data structure representing a single token in the input stream.  This
-// contains an enum for the type, the source position, a GumboStringPiece
-// pointing to the original text, and then a union for any parsed data.
-typedef struct GumboInternalToken {
-  GumboTokenType type;
-  GumboSourcePosition position;
-  GumboStringPiece original_text;
-  union {
-    GumboTokenDocType doc_type;
-    GumboTokenStartTag start_tag;
-    GumboTag end_tag;
-    const char* text;  // For comments.
-    int character;     // For character, whitespace, null, and EOF tokens.
-  } v;
-} GumboToken;
-
-// Initializes the tokenizer state within the GumboParser object, setting up a
-// parse of the specified text.
-void gumbo_tokenizer_state_init(
-    struct GumboInternalParser* parser, const char* text, size_t text_length);
-
-// Destroys the tokenizer state within the GumboParser object, freeing any
-// dynamically-allocated structures within it.
-void gumbo_tokenizer_state_destroy(struct GumboInternalParser* parser);
-
-// Sets the tokenizer state to the specified value.  This is needed by some
-// parser states, which alter the state of the tokenizer in response to tags
-// seen.
-void gumbo_tokenizer_set_state(
-    struct GumboInternalParser* parser, GumboTokenizerEnum state);
-
-// Flags whether the current node is a foreign content element.  This is
-// necessary for the markup declaration open state, where the tokenizer must be
-// aware of the state of the parser to properly tokenize bad comment tags.
-// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#markup-declaration-open-state
-void gumbo_tokenizer_set_is_current_node_foreign(
-    struct GumboInternalParser* parser, bool is_foreign);
-
-// Lexes a single token from the specified buffer, filling the output with the
-// parsed GumboToken data structure.  Returns true for a successful
-// tokenization, false if a parse error occurs.
-//
-// Example:
-//   struct GumboInternalParser parser;
-//   GumboToken output;
-//   gumbo_tokenizer_state_init(&parser, text, strlen(text));
-//   while (gumbo_lex(&parser, &output)) {
-//     ...do stuff with output.
-//     gumbo_token_destroy(&parser, &token);
-//   }
-//   gumbo_tokenizer_state_destroy(&parser);
-bool gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
-
-// Frees the internally-allocated pointers within an GumboToken.  Note that this
-// doesn't free the token itself, since oftentimes it will be allocated on the
-// stack.  A simple call to free() (or GumboParser->deallocator, if
-// appropriate) can handle that.
-//
-// Note that if you are handing over ownership of the internal strings to some
-// other data structure - for example, a parse tree - these do not need to be
-// freed.
-void gumbo_token_destroy(struct GumboInternalParser* parser, GumboToken* token);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // GUMBO_TOKENIZER_H_

data/gumbo-parser/src/tokenizer_states.h

@@ -1,103 +0,0 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains the list of states used in the tokenizer.  Although at first
-// glance it seems like these could be kept internal to the tokenizer, several
-// of the actions in the parser require that it reach into the tokenizer and
-// reset the tokenizer state.  For that to work, it needs to have the
-// definitions of individual states available.
-//
-// This may also be useful for providing more detailed error messages for parse
-// errors, as we can match up states and inputs in a table without having to
-// clutter the tokenizer code with lots of precise error messages.
-
-#ifndef GUMBO_TOKENIZER_STATES_H_
-#define GUMBO_TOKENIZER_STATES_H_
-
-// The ordering of this enum is also used to build the dispatch table for the
-// tokenizer state machine, so if it is changed, be sure to update that too.
-typedef enum {
-  GUMBO_LEX_DATA,
-  GUMBO_LEX_CHAR_REF_IN_DATA,
-  GUMBO_LEX_RCDATA,
-  GUMBO_LEX_CHAR_REF_IN_RCDATA,
-  GUMBO_LEX_RAWTEXT,
-  GUMBO_LEX_SCRIPT,
-  GUMBO_LEX_PLAINTEXT,
-  GUMBO_LEX_TAG_OPEN,
-  GUMBO_LEX_END_TAG_OPEN,
-  GUMBO_LEX_TAG_NAME,
-  GUMBO_LEX_RCDATA_LT,
-  GUMBO_LEX_RCDATA_END_TAG_OPEN,
-  GUMBO_LEX_RCDATA_END_TAG_NAME,
-  GUMBO_LEX_RAWTEXT_LT,
-  GUMBO_LEX_RAWTEXT_END_TAG_OPEN,
-  GUMBO_LEX_RAWTEXT_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_LT,
-  GUMBO_LEX_SCRIPT_END_TAG_OPEN,
-  GUMBO_LEX_SCRIPT_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_ESCAPED_START,
-  GUMBO_LEX_SCRIPT_ESCAPED_START_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED,
-  GUMBO_LEX_SCRIPT_ESCAPED_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED_LT,
-  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN,
-  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END,
-  GUMBO_LEX_BEFORE_ATTR_NAME,
-  GUMBO_LEX_ATTR_NAME,
-  GUMBO_LEX_AFTER_ATTR_NAME,
-  GUMBO_LEX_BEFORE_ATTR_VALUE,
-  GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED,
-  GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED,
-  GUMBO_LEX_ATTR_VALUE_UNQUOTED,
-  GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE,
-  GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED,
-  GUMBO_LEX_SELF_CLOSING_START_TAG,
-  GUMBO_LEX_BOGUS_COMMENT,
-  GUMBO_LEX_MARKUP_DECLARATION,
-  GUMBO_LEX_COMMENT_START,
-  GUMBO_LEX_COMMENT_START_DASH,
-  GUMBO_LEX_COMMENT,
-  GUMBO_LEX_COMMENT_END_DASH,
-  GUMBO_LEX_COMMENT_END,
-  GUMBO_LEX_COMMENT_END_BANG,
-  GUMBO_LEX_DOCTYPE,
-  GUMBO_LEX_BEFORE_DOCTYPE_NAME,
-  GUMBO_LEX_DOCTYPE_NAME,
-  GUMBO_LEX_AFTER_DOCTYPE_NAME,
-  GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD,
-  GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID,
-  GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED,
-  GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED,
-  GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID,
-  GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID,
-  GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD,
-  GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID,
-  GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED,
-  GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED,
-  GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID,
-  GUMBO_LEX_BOGUS_DOCTYPE,
-  GUMBO_LEX_CDATA
-} GumboTokenizerEnum;
-
-#endif  // GUMBO_TOKENIZER_STATES_H_

data/gumbo-parser/src/utf8.c

@@ -1,270 +0,0 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-
-#include "utf8.h"
-
-#include <assert.h>
-#include <stdint.h>
-#include <string.h>
-#include <strings.h>  // For strncasecmp.
-
-#include "error.h"
-#include "gumbo.h"
-#include "parser.h"
-#include "util.h"
-#include "vector.h"
-
-const int kUtf8ReplacementChar = 0xFFFD;
-
-// Reference material:
-// Wikipedia: http://en.wikipedia.org/wiki/UTF-8#Description
-// RFC 3629: http://tools.ietf.org/html/rfc3629
-// HTML5 Unicode handling:
-// http://www.whatwg.org/specs/web-apps/current-work/multipage/syntax.html#preprocessing-the-input-stream
-//
-// This implementation is based on a DFA-based decoder by Bjoern Hoehrmann
-// <bjoern@hoehrmann.de>.  We wrap the inner table-based decoder routine in our
-// own handling for newlines, tabs, invalid continuation bytes, and other
-// conditions that the HTML5 spec fully specifies but normal UTF8 decoders do
-// not handle.
-// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.  Full text of
-// the license agreement and code follows.
-
-// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
-
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to
-// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-// of the Software, and to permit persons to whom the Software is furnished to
-// do
-// so, subject to the following conditions:
-
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-
-#define UTF8_ACCEPT 0
-#define UTF8_REJECT 12
-
-static const uint8_t utf8d[] = {
-    // The first part of the table maps bytes to character classes that
-    // to reduce the size of the transition table and create bitmasks.
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9,
-    9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
-    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 2, 2, 2, 2, 2, 2,
-    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10,
-    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8,
-    8, 8, 8, 8, 8, 8,
-
-    // The second part is a transition table that maps a combination
-    // of a state of the automaton and a character class to a state.
-    0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12,
-    12, 12, 12, 12, 12, 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12,
-    12, 12, 12, 12, 24, 12, 24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12,
-    12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12,
-    12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12,
-    12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
-};
-
-uint32_t static inline decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
-  uint32_t type = utf8d[byte];
-
-  *codep = (*state != UTF8_ACCEPT) ? (byte & 0x3fu) | (*codep << 6)
-                                   : (0xff >> type) & (byte);
-
-  *state = utf8d[256 + *state + type];
-  return *state;
-}
-
-// END COPIED CODE.
-
-// Adds a decoding error to the parser's error list, based on the current state
-// of the Utf8Iterator.
-static void add_error(Utf8Iterator* iter, GumboErrorType type) {
-  GumboParser* parser = iter->_parser;
-
-  GumboError* error = gumbo_add_error(parser);
-  if (!error) {
-    return;
-  }
-  error->type = type;
-  error->position = iter->_pos;
-  error->original_text = iter->_start;
-
-  // At the point the error is recorded, the code point hasn't been computed
-  // yet (and can't be, because it's invalid), so we need to build up the raw
-  // hex value from the bytes under the cursor.
-  uint64_t code_point = 0;
-  for (int i = 0; i < iter->_width; ++i) {
-    code_point = (code_point << 8) | (unsigned char) iter->_start[i];
-  }
-  error->v.codepoint = code_point;
-}
-
-// Reads the next UTF-8 character in the iter.
-// This assumes that iter->_start points to the beginning of the character.
-// When this method returns, iter->_width and iter->_current will be set
-// appropriately, as well as any error flags.
-static void read_char(Utf8Iterator* iter) {
-  if (iter->_start >= iter->_end) {
-    // No input left to consume; emit an EOF and set width = 0.
-    iter->_current = -1;
-    iter->_width = 0;
-    return;
-  }
-
-  uint32_t code_point = 0;
-  uint32_t state = UTF8_ACCEPT;
-  for (const char* c = iter->_start; c < iter->_end; ++c) {
-    decode(&state, &code_point, (uint32_t)(unsigned char) (*c));
-    if (state == UTF8_ACCEPT) {
-      iter->_width = c - iter->_start + 1;
-      // This is the special handling for carriage returns that is mandated by
-      // the HTML5 spec.  Since we're looking for particular 7-bit literal
-      // characters, we operate in terms of chars and only need a check for iter
-      // overrun, instead of having to read in a full next code point.
-      // http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#preprocessing-the-input-stream
-      if (code_point == '\r') {
-        assert(iter->_width == 1);
-        const char* next = c + 1;
-        if (next < iter->_end && *next == '\n') {
-          // Advance the iter, as if the carriage return didn't exist.
-          ++iter->_start;
-          // Preserve the true offset, since other tools that look at it may be
-          // unaware of HTML5's rules for converting \r into \n.
-          ++iter->_pos.offset;
-        }
-        code_point = '\n';
-      }
-      if (utf8_is_invalid_code_point(code_point)) {
-        add_error(iter, GUMBO_ERR_UTF8_INVALID);
-        code_point = kUtf8ReplacementChar;
-      }
-      iter->_current = code_point;
-      return;
-    } else if (state == UTF8_REJECT) {
-      // We don't want to consume the invalid continuation byte of a multi-byte
-      // run, but we do want to skip past an invalid first byte.
-      iter->_width = c - iter->_start + (c == iter->_start);
-      iter->_current = kUtf8ReplacementChar;
-      add_error(iter, GUMBO_ERR_UTF8_INVALID);
-      return;
-    }
-  }
-  // If we got here without exiting early, then we've reached the end of the
-  // iterator.  Add an error for truncated input, set the width to consume the
-  // rest of the iterator, and emit a replacement character.  The next time we
-  // enter this method, it will detect that there's no input to consume and
-  // output an EOF.
-  iter->_current = kUtf8ReplacementChar;
-  iter->_width = iter->_end - iter->_start;
-  add_error(iter, GUMBO_ERR_UTF8_TRUNCATED);
-}
-
-static void update_position(Utf8Iterator* iter) {
-  iter->_pos.offset += iter->_width;
-  if (iter->_current == '\n') {
-    ++iter->_pos.line;
-    iter->_pos.column = 1;
-  } else if (iter->_current == '\t') {
-    int tab_stop = iter->_parser->_options->tab_stop;
-    iter->_pos.column = ((iter->_pos.column / tab_stop) + 1) * tab_stop;
-  } else if (iter->_current != -1) {
-    ++iter->_pos.column;
-  }
-}
-
-// Returns true if this Unicode code point is in the list of characters
-// forbidden by the HTML5 spec, such as undefined control chars.
-bool utf8_is_invalid_code_point(int c) {
-  return (c >= 0x1 && c <= 0x8) || c == 0xB || (c >= 0xE && c <= 0x1F) ||
-         (c >= 0x7F && c <= 0x9F) || (c >= 0xFDD0 && c <= 0xFDEF) ||
-         ((c & 0xFFFF) == 0xFFFE) || ((c & 0xFFFF) == 0xFFFF);
-}
-
-void utf8iterator_init(GumboParser* parser, const char* source,
-    size_t source_length, Utf8Iterator* iter) {
-  iter->_start = source;
-  iter->_end = source + source_length;
-  iter->_pos.line = 1;
-  iter->_pos.column = 1;
-  iter->_pos.offset = 0;
-  iter->_parser = parser;
-  read_char(iter);
-}
-
-void utf8iterator_next(Utf8Iterator* iter) {
-  // We update positions based on the *last* character read, so that the first
-  // character following a newline is at column 1 in the next line.
-  update_position(iter);
-  iter->_start += iter->_width;
-  read_char(iter);
-}
-
-int utf8iterator_current(const Utf8Iterator* iter) { return iter->_current; }
-
-void utf8iterator_get_position(
-    const Utf8Iterator* iter, GumboSourcePosition* output) {
-  *output = iter->_pos;
-}
-
-const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter) {
-  return iter->_start;
-}
-
-const char* utf8iterator_get_end_pointer(const Utf8Iterator* iter) {
-  return iter->_end;
-}
-
-bool utf8iterator_maybe_consume_match(Utf8Iterator* iter, const char* prefix,
-    size_t length, bool case_sensitive) {
-  bool matched = (iter->_start + length <= iter->_end) &&
-                 (case_sensitive ? !strncmp(iter->_start, prefix, length)
-                                 : !strncasecmp(iter->_start, prefix, length));
-  if (matched) {
-    for (unsigned int i = 0; i < length; ++i) {
-      utf8iterator_next(iter);
-    }
-    return true;
-  } else {
-    return false;
-  }
-}
-
-void utf8iterator_mark(Utf8Iterator* iter) {
-  iter->_mark = iter->_start;
-  iter->_mark_pos = iter->_pos;
-}
-
-// Returns the current input stream position to the mark.
-void utf8iterator_reset(Utf8Iterator* iter) {
-  iter->_start = iter->_mark;
-  iter->_pos = iter->_mark_pos;
-  read_char(iter);
-}
-
-// Sets the position and original text fields of an error to the value at the
-// mark.
-void utf8iterator_fill_error_at_mark(Utf8Iterator* iter, GumboError* error) {
-  error->position = iter->_mark_pos;
-  error->original_text = iter->_mark;
-}