nokogumbo 0.4 → 0.5

data/work/tokenizer.h

@@ -1,123 +0,0 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains an implementation of a tokenizer for HTML5.  It consumes a
-// buffer of UTF-8 characters, and then emits a stream of tokens.
-
-#ifndef GUMBO_TOKENIZER_H_
-#define GUMBO_TOKENIZER_H_
-
-#include <stdbool.h>
-#include <stddef.h>
-
-#include "gumbo.h"
-#include "token_type.h"
-#include "tokenizer_states.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct _GumboParser;
-
-// Struct containing all information pertaining to doctype tokens.
-typedef struct _GumboTokenDocType {
-  const char* name;
-  const char* public_identifier;
-  const char* system_identifier;
-  bool force_quirks;
-  // There's no way to tell a 0-length public or system ID apart from the
-  // absence of a public or system ID, but they're handled different by the
-  // spec, so we need bool flags for them.
-  bool has_public_identifier;
-  bool has_system_identifier;
-} GumboTokenDocType;
-
-// Struct containing all information pertaining to start tag tokens.
-typedef struct _GumboTokenStartTag {
-  GumboTag tag;
-  GumboVector /* GumboAttribute */ attributes;
-  bool is_self_closing;
-} GumboTokenStartTag;
-
-// A data structure representing a single token in the input stream.  This
-// contains an enum for the type, the source position, a GumboStringPiece
-// pointing to the original text, and then a union for any parsed data.
-typedef struct _GumboToken {
-  GumboTokenType type;
-  GumboSourcePosition position;
-  GumboStringPiece original_text;
-  union {
-    GumboTokenDocType doc_type;
-    GumboTokenStartTag start_tag;
-    GumboTag end_tag;
-    const char* text;    // For comments.
-    int character;      // For character, whitespace, null, and EOF tokens.
-  } v;
-} GumboToken;
-
-// Initializes the tokenizer state within the GumboParser object, setting up a
-// parse of the specified text.
-void gumbo_tokenizer_state_init(
-    struct _GumboParser* parser, const char* text, size_t text_length);
-
-// Destroys the tokenizer state within the GumboParser object, freeing any
-// dynamically-allocated structures within it.
-void gumbo_tokenizer_state_destroy(struct _GumboParser* parser);
-
-// Sets the tokenizer state to the specified value.  This is needed by some
-// parser states, which alter the state of the tokenizer in response to tags
-// seen.
-void gumbo_tokenizer_set_state(
-    struct _GumboParser* parser, GumboTokenizerEnum state);
-
-// Flags whether the current node is a foreign content element.  This is
-// necessary for the markup declaration open state, where the tokenizer must be
-// aware of the state of the parser to properly tokenize bad comment tags.
-// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#markup-declaration-open-state
-void gumbo_tokenizer_set_is_current_node_foreign(
-    struct _GumboParser* parser, bool is_foreign);
-
-// Lexes a single token from the specified buffer, filling the output with the
-// parsed GumboToken data structure.  Returns true for a successful
-// tokenization, false if a parse error occurs.
-//
-// Example:
-//   struct _GumboParser parser;
-//   GumboToken output;
-//   gumbo_tokenizer_state_init(&parser, text, strlen(text));
-//   while (gumbo_lex(&parser, &output)) {
-//     ...do stuff with output.
-//     gumbo_token_destroy(&parser, &token);
-//   }
-//   gumbo_tokenizer_state_destroy(&parser);
-bool gumbo_lex(struct _GumboParser* parser, GumboToken* output);
-
-// Frees the internally-allocated pointers within an GumboToken.  Note that this
-// doesn't free the token itself, since oftentimes it will be allocated on the
-// stack.  A simple call to free() (or struct _GumboParser->deallocator, if
-// appropriate) can handle that.
-//
-// Note that if you are handing over ownership of the internal strings to some
-// other data structure - for example, a parse tree - these do not need to be
-// freed.
-void gumbo_token_destroy(struct _GumboParser* parser, GumboToken* token);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // GUMBO_TOKENIZER_H_

data/work/tokenizer_states.h

@@ -1,103 +0,0 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains the list of states used in the tokenizer.  Although at first
-// glance it seems like these could be kept internal to the tokenizer, several
-// of the actions in the parser require that it reach into the tokenizer and
-// reset the tokenizer state.  For that to work, it needs to have the
-// definitions of individual states available.
-//
-// This may also be useful for providing more detailed error messages for parse
-// errors, as we can match up states and inputs in a table without having to
-// clutter the tokenizer code with lots of precise error messages.
-
-#ifndef GUMBO_TOKENIZER_STATES_H_
-#define GUMBO_TOKENIZER_STATES_H_
-
-// The ordering of this enum is also used to build the dispatch table for the
-// tokenizer state machine, so if it is changed, be sure to update that too.
-typedef enum _GumboTokenizerEnum {
-  GUMBO_LEX_DATA,
-  GUMBO_LEX_CHAR_REF_IN_DATA,
-  GUMBO_LEX_RCDATA,
-  GUMBO_LEX_CHAR_REF_IN_RCDATA,
-  GUMBO_LEX_RAWTEXT,
-  GUMBO_LEX_SCRIPT,
-  GUMBO_LEX_PLAINTEXT,
-  GUMBO_LEX_TAG_OPEN,
-  GUMBO_LEX_END_TAG_OPEN,
-  GUMBO_LEX_TAG_NAME,
-  GUMBO_LEX_RCDATA_LT,
-  GUMBO_LEX_RCDATA_END_TAG_OPEN,
-  GUMBO_LEX_RCDATA_END_TAG_NAME,
-  GUMBO_LEX_RAWTEXT_LT,
-  GUMBO_LEX_RAWTEXT_END_TAG_OPEN,
-  GUMBO_LEX_RAWTEXT_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_LT,
-  GUMBO_LEX_SCRIPT_END_TAG_OPEN,
-  GUMBO_LEX_SCRIPT_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_ESCAPED_START,
-  GUMBO_LEX_SCRIPT_ESCAPED_START_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED,
-  GUMBO_LEX_SCRIPT_ESCAPED_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH,
-  GUMBO_LEX_SCRIPT_ESCAPED_LT,
-  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN,
-  GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT,
-  GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END,
-  GUMBO_LEX_BEFORE_ATTR_NAME,
-  GUMBO_LEX_ATTR_NAME,
-  GUMBO_LEX_AFTER_ATTR_NAME,
-  GUMBO_LEX_BEFORE_ATTR_VALUE,
-  GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED,
-  GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED,
-  GUMBO_LEX_ATTR_VALUE_UNQUOTED,
-  GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE,
-  GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED,
-  GUMBO_LEX_SELF_CLOSING_START_TAG,
-  GUMBO_LEX_BOGUS_COMMENT,
-  GUMBO_LEX_MARKUP_DECLARATION,
-  GUMBO_LEX_COMMENT_START,
-  GUMBO_LEX_COMMENT_START_DASH,
-  GUMBO_LEX_COMMENT,
-  GUMBO_LEX_COMMENT_END_DASH,
-  GUMBO_LEX_COMMENT_END,
-  GUMBO_LEX_COMMENT_END_BANG,
-  GUMBO_LEX_DOCTYPE,
-  GUMBO_LEX_BEFORE_DOCTYPE_NAME,
-  GUMBO_LEX_DOCTYPE_NAME,
-  GUMBO_LEX_AFTER_DOCTYPE_NAME,
-  GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD,
-  GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID,
-  GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED,
-  GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED,
-  GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID,
-  GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID,
-  GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD,
-  GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID,
-  GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED,
-  GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED,
-  GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID,
-  GUMBO_LEX_BOGUS_DOCTYPE,
-  GUMBO_LEX_CDATA
-} GumboTokenizerEnum;
-
-#endif  // GUMBO_TOKENIZER_STATES_H_

data/work/utf8.c

@@ -1,268 +0,0 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-
-#include "utf8.h"
-
-#include <assert.h>
-#include <stdint.h>
-#include <string.h>
-#include <strings.h>    // For strncasecmp.
-
-#include "error.h"
-#include "gumbo.h"
-#include "parser.h"
-#include "util.h"
-#include "vector.h"
-
-const int kUtf8ReplacementChar = 0xFFFD;
-
-// Reference material:
-// Wikipedia: http://en.wikipedia.org/wiki/UTF-8#Description
-// RFC 3629: http://tools.ietf.org/html/rfc3629
-// HTML5 Unicode handling:
-// http://www.whatwg.org/specs/web-apps/current-work/multipage/infrastructure.html#decoded-as-utf-8,-with-error-handling
-
-// Adds a decoding error to the parser's error list, based on the current state
-// of the Utf8Iterator.
-static void add_error(Utf8Iterator* iter, GumboErrorType type) {
-  GumboParser* parser = iter->_parser;
-
-  GumboError* error = gumbo_add_error(parser);
-  if (!error) {
-    return;
-  }
-  error->type = type;
-  error->position = iter->_pos;
-  error->original_text = iter->_start;
-
-  // At the point the error is recorded, the code point hasn't been computed
-  // yet (and can't be, because it's invalid), so we need to build up the raw
-  // hex value from the bytes under the cursor.
-  uint64_t code_point = 0;
-  for (int i = 0; i < iter->_width; ++i) {
-    code_point = (code_point << 8) | (unsigned char) iter->_start[i];
-  }
-  error->v.codepoint = code_point;
-}
-
-// Reads the next UTF-8 character in the iter.
-// This assumes that iter->_start points to the beginning of the character.
-// When this method returns, iter->_width and iter->_current will be set
-// appropriately, as well as any error flags.
-static void read_char(Utf8Iterator* iter) {
-  unsigned char c;
-  unsigned char mask = '\0';
-  int is_bad_char = false;
-
-  c = (unsigned char) *iter->_start;
-  if (c < 0x80) {
-    // Valid one-byte sequence.
-    iter->_width = 1;
-    mask = 0xFF;
-  } else if (c < 0xC0) {
-    // Continuation character not following a multibyte sequence.
-    // The HTML5 spec here says to consume the byte and output a replacement
-    // character.
-    iter->_width = 1;
-    is_bad_char = true;
-  } else if (c < 0xE0) {
-    iter->_width = 2;
-    mask = 0x1F;                // 00011111 in binary.
-    if (c < 0xC2) {
-      // Overlong encoding; error according to UTF8/HTML5 spec.
-      is_bad_char = true;
-    }
-  } else if (c < 0xF0) {
-    iter->_width = 3;
-    mask = 0xF;                 // 00001111 in binary.
-  } else if (c < 0xF5) {
-    iter->_width = 4;
-    mask = 0x7;                 // 00000111 in binary.
-  } else if (c < 0xF8) {
-    // The following cases are all errors, but we need to handle them separately
-    // so that we consume the proper number of bytes from the input stream
-    // before replacing them with the replacement char.  The HTML5 spec
-    // specifies that we should consume the shorter of the length specified by
-    // the first bit or the run leading up to the first non-continuation
-    // character.
-    iter->_width = 5;
-    is_bad_char = true;
-  } else if (c < 0xFC) {
-    iter->_width = 6;
-    is_bad_char = true;
-  } else if (c < 0xFE) {
-    iter->_width = 7;
-    is_bad_char = true;
-  } else {
-    iter->_width = 1;
-    is_bad_char = true;
-  }
-
-  // Check to make sure we have enough bytes left in the iter to read all that
-  // we want.  If not, we set the iter_truncated flag, mark this as a bad
-  // character, and adjust the current width so that it consumes the rest of the
-  // iter.
-  int code_point = c & mask;
-  if (iter->_start + iter->_width > iter->_end) {
-    iter->_width = iter->_end - iter->_start;
-    add_error(iter, GUMBO_ERR_UTF8_TRUNCATED);
-    is_bad_char = true;
-  }
-
-  // Now we decode continuation bytes, shift them appropriately, and build up
-  // the appropriate code point.
-  assert(iter->_width < 8);
-  for (int i = 1; i < iter->_width; ++i) {
-    c = (unsigned char) iter->_start[i];
-    if (c < 0x80 || c > 0xBF) {
-      // Per HTML5 spec, we don't include the invalid continuation char in the
-      // run that we consume here.
-      iter->_width = i;
-      is_bad_char = true;
-      break;
-    }
-    code_point = (code_point << 6) | (c & ~0x80);
-  }
-
-  // If we had a decode error, set the current code point to the replacement
-  // character and flip the flag indicating that a decode error occurred.
-  // Ditto if we have a code point that is explicitly on the list of characters
-  // prohibited by the HTML5 spec, such as control characters.
-  if (is_bad_char || utf8_is_invalid_code_point(code_point)) {
-    add_error(iter, GUMBO_ERR_UTF8_INVALID);
-    code_point = kUtf8ReplacementChar;
-  }
-
-  // This is the special handling for carriage returns that is mandated by the
-  // HTML5 spec.  Since we're looking for particular 7-bit literal characters,
-  // we operate in terms of chars and only need a check for iter overrun,
-  // instead of having to read in a full next code point.
-  // http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#preprocessing-the-input-stream
-  if (code_point == '\r') {
-    const char* next = iter->_start + iter->_width;
-    if (next < iter->_end && *next == '\n') {
-      // Advance the iter, as if the carriage return didn't exist.
-      ++iter->_start;
-      // Preserve the true offset, since other tools that look at it may be
-      // unaware of HTML5's rules for converting \r into \n.
-      ++iter->_pos.offset;
-    }
-    code_point = '\n';
-  }
-
-  // At this point, we know we have a valid character as the code point, so we
-  // set it, and we're done.
-  iter->_current = code_point;
-}
-
-static void update_position(Utf8Iterator* iter) {
-  iter->_pos.offset += iter->_width;
-  if (iter->_current == '\n') {
-    ++iter->_pos.line;
-    iter->_pos.column = 1;
-  } else if(iter->_current == '\t') {
-    int tab_stop = iter->_parser->_options->tab_stop;
-    iter->_pos.column = ((iter->_pos.column / tab_stop) + 1) * tab_stop;
-  } else {
-    ++iter->_pos.column;
-  }
-}
-
-// Returns true if this Unicode code point is in the list of characters
-// forbidden by the HTML5 spec, such as undefined control chars.
-bool utf8_is_invalid_code_point(int c) {
-  return (c >= 0x1 && c <= 0x8) || c == 0xB || (c >= 0xE && c <= 0x1F) ||
-      (c >= 0x7F && c <= 0x9F) || (c >= 0xFDD0 && c <= 0xFDEF) ||
-      ((c & 0xFFFF) == 0xFFFE) || ((c & 0xFFFF) == 0xFFFF);
-}
-
-void utf8iterator_init(
-    GumboParser* parser, const char* source, size_t source_length,
-    Utf8Iterator* iter) {
-  iter->_start = source;
-  iter->_end = source + source_length;
-  iter->_width = 0;
-  iter->_pos.line = 1;
-  iter->_pos.column = 1;
-  iter->_pos.offset = 0;
-  iter->_parser = parser;
-  if (source_length) {
-    read_char(iter);
-  } else {
-    iter->_current = -1;
-  }
-}
-
-void utf8iterator_next(Utf8Iterator* iter) {
-  iter->_start += iter->_width;
-  // We update positions based on the *last* character read, so that the first
-  // character following a newline is at column 1 in the next line.
-  update_position(iter);
-  if (iter->_start < iter->_end) {
-    read_char(iter);
-  } else {  // EOF
-    iter->_current = -1;
-  }
-}
-
-int utf8iterator_current(const Utf8Iterator* iter) {
-  return iter->_current;
-}
-
-void utf8iterator_get_position(
-    const Utf8Iterator* iter, GumboSourcePosition* output) {
-  *output = iter->_pos;
-}
-
-const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter) {
-  return iter->_start;
-}
-
-bool utf8iterator_maybe_consume_match(
-    Utf8Iterator* iter, const char* prefix, size_t length,
-    bool case_sensitive) {
-  bool matched = (iter->_start + length <= iter->_end) && (case_sensitive ?
-      !strncmp(iter->_start, prefix, length) :
-      !strncasecmp(iter->_start, prefix, length));
-  if (matched) {
-    for (int i = 0; i < length; ++i) {
-      utf8iterator_next(iter);
-    }
-    return true;
-  } else {
-    return false;
-  }
-}
-
-void utf8iterator_mark(Utf8Iterator* iter) {
-  iter->_mark = iter->_start;
-  iter->_mark_pos = iter->_pos;
-}
-
-// Returns the current input stream position to the mark.
-void utf8iterator_reset(Utf8Iterator* iter) {
-  iter->_start = iter->_mark;
-  iter->_pos = iter->_mark_pos;
-  read_char(iter);
-}
-
-// Sets the position and original text fields of an error to the value at the
-// mark.
-void utf8iterator_fill_error_at_mark(
-    Utf8Iterator* iter, GumboError* error) {
-  error->position = iter->_mark_pos;
-  error->original_text = iter->_mark;
-}