nokogumbo 0.5 → 0.5.1

data/work/char_ref.h

@@ -0,0 +1,61 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// Internal header for character reference handling; this should not be exposed
+// transitively by any public API header.  This is why the functions aren't
+// namespaced.
+
+#ifndef GUMBO_CHAR_REF_H_
+#define GUMBO_CHAR_REF_H_
+
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct _GumboParser;
+struct _Utf8Iterator;
+
+// Value that indicates no character was produced.
+extern const int kGumboNoChar;
+
+// Certain named character references generate two codepoints, not one, and so
+// the consume_char_ref subroutine needs to return this instead of an int.  The
+// first field will be kGumboNoChar if no character reference was found; the
+// second field will be kGumboNoChar if that is the case or if the character
+// reference returns only a single codepoint.
+typedef struct {
+  int first;
+  int second;
+} OneOrTwoCodepoints;
+
+// Implements the "consume a character reference" section of the spec.
+// This reads in characters from the input as necessary, and fills in a
+// OneOrTwoCodepoints struct containing the characters read.  It may add parse
+// errors to the GumboParser's errors vector, if the spec calls for it.  Pass a
+// space for the "additional allowed char" when the spec says "with no
+// additional allowed char".  Returns false on parse error, true otherwise.
+bool consume_char_ref(
+    struct _GumboParser* parser, struct _Utf8Iterator* input,
+    int additional_allowed_char, bool is_in_attribute,
+    OneOrTwoCodepoints* output);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_CHAR_REF_H_

data/work/error.c

@@ -0,0 +1,258 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "error.h"
+
+#include <assert.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "gumbo.h"
+#include "parser.h"
+#include "string_buffer.h"
+#include "util.h"
+#include "vector.h"
+
+static const size_t kMessageBufferSize = 256;
+
+// Prints a formatted message to a StringBuffer.  This automatically resizes the
+// StringBuffer as necessary to fit the message.  Returns the number of bytes
+// written.
+static int print_message(GumboParser* parser, GumboStringBuffer* output,
+                         const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  int remaining_capacity = output->capacity - output->length;
+  int bytes_written = vsnprintf(output->data + output->length,
+                                remaining_capacity, format, args);
+  if (bytes_written > remaining_capacity) {
+    gumbo_string_buffer_reserve(
+        parser, output->capacity + bytes_written, output);
+    remaining_capacity = output->capacity - output->length;
+    bytes_written = vsnprintf(output->data + output->length,
+                              remaining_capacity, format, args);
+  }
+  output->length += bytes_written;
+  va_end(args);
+  return bytes_written;
+}
+
+static void print_tag_stack(
+    GumboParser* parser, const GumboParserError* error,
+    GumboStringBuffer* output) {
+  print_message(parser, output, "  Currently open tags: ");
+  for (int i = 0; i < error->tag_stack.length; ++i) {
+    if (i) {
+      print_message(parser, output, ", ");
+    }
+    GumboTag tag = (GumboTag) error->tag_stack.data[i];
+    print_message(parser, output, gumbo_normalized_tagname(tag));
+  }
+  gumbo_string_buffer_append_codepoint(parser, '.', output);
+}
+
+static void handle_parser_error(GumboParser* parser,
+                                const GumboParserError* error,
+                                GumboStringBuffer* output) {
+  if (error->parser_state == GUMBO_INSERTION_MODE_INITIAL &&
+      error->input_type != GUMBO_TOKEN_DOCTYPE) {
+    print_message(parser, output,
+                  "The doctype must be the first token in the document");
+    return;
+  }
+
+  switch (error->input_type) {
+    case GUMBO_TOKEN_DOCTYPE:
+      print_message(parser, output, "This is not a legal doctype");
+      return;
+    case GUMBO_TOKEN_COMMENT:
+      // Should never happen; comments are always legal.
+      assert(0);
+      // But just in case...
+      print_message(parser, output, "Comments aren't legal here");
+      return;
+    case GUMBO_TOKEN_WHITESPACE:
+    case GUMBO_TOKEN_CHARACTER:
+      print_message(parser, output, "Character tokens aren't legal here");
+      return;
+    case GUMBO_TOKEN_NULL:
+      print_message(parser, output, "Null bytes are not allowed in HTML5");
+      return;
+    case GUMBO_TOKEN_EOF:
+      if (error->parser_state == GUMBO_INSERTION_MODE_INITIAL) {
+        print_message(parser, output, "You must provide a doctype");
+      } else {
+        print_message(parser, output, "Premature end of file");
+        print_tag_stack(parser, error, output);
+      }
+      return;
+    case GUMBO_TOKEN_START_TAG:
+    case GUMBO_TOKEN_END_TAG:
+      print_message(parser, output, "That tag isn't allowed here");
+      print_tag_stack(parser, error, output);
+      // TODO(jdtang): Give more specific messaging.
+      return;
+  }
+}
+
+// Finds the preceding newline in an original source buffer from a given byte
+// location.  Returns a character pointer to the character after that, or a
+// pointer to the beginning of the string if this is the first line.
+static const char* find_last_newline(
+    const char* original_text, const char* error_location) {
+  assert(error_location >= original_text);
+  const char* c = error_location;
+  for (; c != original_text && *c != '\n'; --c) {
+    // There may be an error at EOF, which would be a nul byte.
+    assert(*c || c == error_location);
+  }
+  return c == original_text ? c : c + 1;
+}
+
+// Finds the next newline in the original source buffer from a given byte
+// location.  Returns a character pointer to that newline, or a pointer to the
+// terminating null byte if this is the last line.
+static const char* find_next_newline(
+    const char* original_text, const char* error_location) {
+  const char* c = error_location;
+  for (; *c && *c != '\n'; ++c);
+  return c;
+}
+
+GumboError* gumbo_add_error(GumboParser* parser) {
+  int max_errors = parser->_options->max_errors;
+  if (max_errors < 0 && parser->_output->errors.length >= max_errors) {
+    return NULL;
+  }
+  GumboError* error = gumbo_parser_allocate(parser, sizeof(GumboError));
+  gumbo_vector_add(parser, error, &parser->_output->errors);
+  return error;
+}
+
+void gumbo_error_to_string(
+    GumboParser* parser, const GumboError* error, GumboStringBuffer* output) {
+  print_message(parser, output, "@%d:%d: ",
+                error->position.line, error->position.column);
+  switch (error->type) {
+    case GUMBO_ERR_UTF8_INVALID:
+      print_message(parser, output, "Invalid UTF8 character 0x%x",
+               error->v.codepoint);
+      break;
+    case GUMBO_ERR_UTF8_TRUNCATED:
+      print_message(parser, output,
+               "Input stream ends with a truncated UTF8 character 0x%x",
+               error->v.codepoint);
+      break;
+    case GUMBO_ERR_NUMERIC_CHAR_REF_NO_DIGITS:
+      print_message(parser, output,
+               "No digits after &# in numeric character reference");
+      break;
+    case GUMBO_ERR_NUMERIC_CHAR_REF_WITHOUT_SEMICOLON:
+      print_message(parser, output,
+               "The numeric character reference &#%d should be followed "
+               "by a semicolon", error->v.codepoint);
+      break;
+    case GUMBO_ERR_NUMERIC_CHAR_REF_INVALID:
+      print_message(parser, output,
+               "The numeric character reference &#%d; encodes an invalid "
+               "unicode codepoint", error->v.codepoint);
+      break;
+    case GUMBO_ERR_NAMED_CHAR_REF_WITHOUT_SEMICOLON:
+      // The textual data came from one of the literal strings in the table, and
+      // so it'll be null-terminated.
+      print_message(parser, output,
+               "The named character reference &%.*s should be followed by a "
+               "semicolon", (int) error->v.text.length, error->v.text.data);
+      break;
+    case GUMBO_ERR_NAMED_CHAR_REF_INVALID:
+      print_message(parser, output,
+               "The named character reference &%.*s; is not a valid entity name",
+               (int) error->v.text.length, error->v.text.data);
+      break;
+    case GUMBO_ERR_DUPLICATE_ATTR:
+      print_message(parser, output,
+               "Attribute %s occurs multiple times, at positions %d and %d",
+               error->v.duplicate_attr.name,
+               error->v.duplicate_attr.original_index,
+               error->v.duplicate_attr.new_index);
+      break;
+    case GUMBO_ERR_PARSER:
+    case GUMBO_ERR_UNACKNOWLEDGED_SELF_CLOSING_TAG:
+      handle_parser_error(parser, &error->v.parser, output);
+      break;
+    default:
+      print_message(parser, output,
+               "Tokenizer error with an unimplemented error message");
+      break;
+  }
+  gumbo_string_buffer_append_codepoint(parser, '.', output);
+}
+
+void gumbo_caret_diagnostic_to_string(
+    GumboParser* parser, const GumboError* error,
+    const char* source_text, GumboStringBuffer* output) {
+  gumbo_error_to_string(parser, error, output);
+
+  const char* line_start =
+      find_last_newline(source_text, error->original_text);
+  const char* line_end =
+      find_next_newline(source_text, error->original_text);
+  GumboStringPiece original_line;
+  original_line.data = line_start;
+  original_line.length = line_end - line_start;
+
+  gumbo_string_buffer_append_codepoint(parser, '\n', output);
+  gumbo_string_buffer_append_string(parser, &original_line, output);
+  gumbo_string_buffer_append_codepoint(parser, '\n', output);
+  gumbo_string_buffer_reserve(
+      parser, output->length + error->position.column, output);
+  int num_spaces = error->position.column - 1;
+  memset(output->data + output->length, ' ', num_spaces);
+  output->length += num_spaces;
+  gumbo_string_buffer_append_codepoint(parser, '^', output);
+  gumbo_string_buffer_append_codepoint(parser, '\n', output);
+}
+
+void gumbo_print_caret_diagnostic(
+    GumboParser* parser, const GumboError* error, const char* source_text) {
+  GumboStringBuffer text;
+  gumbo_string_buffer_init(parser, &text);
+  gumbo_caret_diagnostic_to_string(parser, error, source_text, &text);
+  printf("%.*s", (int) text.length, text.data);
+  gumbo_string_buffer_destroy(parser, &text);
+}
+
+void gumbo_error_destroy(GumboParser* parser, GumboError* error) {
+  if (error->type == GUMBO_ERR_PARSER ||
+      error->type == GUMBO_ERR_UNACKNOWLEDGED_SELF_CLOSING_TAG) {
+    gumbo_vector_destroy(parser, &error->v.parser.tag_stack);
+  } else if (error->type == GUMBO_ERR_DUPLICATE_ATTR) {
+    gumbo_parser_deallocate(parser, (void*) error->v.duplicate_attr.name);
+  }
+  gumbo_parser_deallocate(parser, error);
+}
+
+void gumbo_init_errors(GumboParser* parser) {
+  gumbo_vector_init(parser, 5, &parser->_output->errors);
+}
+
+void gumbo_destroy_errors(GumboParser* parser) {
+  for (int i = 0; i < parser->_output->errors.length; ++i) {
+    gumbo_error_destroy(parser, parser->_output->errors.data[i]);
+  }
+  gumbo_vector_destroy(parser, &parser->_output->errors);
+}

data/work/error.h

@@ -0,0 +1,225 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// Error types, enums, and handling functions.
+
+#ifndef GUMBO_ERROR_H_
+#define GUMBO_ERROR_H_
+
+#include <stdint.h>
+
+#include "gumbo.h"
+#include "insertion_mode.h"
+#include "string_buffer.h"
+#include "token_type.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct _GumboParser;
+
+typedef enum {
+  GUMBO_ERR_UTF8_INVALID,
+  GUMBO_ERR_UTF8_TRUNCATED,
+  GUMBO_ERR_UTF8_NULL,
+  GUMBO_ERR_NUMERIC_CHAR_REF_NO_DIGITS,
+  GUMBO_ERR_NUMERIC_CHAR_REF_WITHOUT_SEMICOLON,
+  GUMBO_ERR_NUMERIC_CHAR_REF_INVALID,
+  GUMBO_ERR_NAMED_CHAR_REF_WITHOUT_SEMICOLON,
+  GUMBO_ERR_NAMED_CHAR_REF_INVALID,
+  GUMBO_ERR_TAG_STARTS_WITH_QUESTION,
+  GUMBO_ERR_TAG_EOF,
+  GUMBO_ERR_TAG_INVALID,
+  GUMBO_ERR_CLOSE_TAG_EMPTY,
+  GUMBO_ERR_CLOSE_TAG_EOF,
+  GUMBO_ERR_CLOSE_TAG_INVALID,
+  GUMBO_ERR_SCRIPT_EOF,
+  GUMBO_ERR_ATTR_NAME_EOF,
+  GUMBO_ERR_ATTR_NAME_INVALID,
+  GUMBO_ERR_ATTR_DOUBLE_QUOTE_EOF,
+  GUMBO_ERR_ATTR_SINGLE_QUOTE_EOF,
+  GUMBO_ERR_ATTR_UNQUOTED_EOF,
+  GUMBO_ERR_ATTR_UNQUOTED_RIGHT_BRACKET,
+  GUMBO_ERR_ATTR_UNQUOTED_EQUALS,
+  GUMBO_ERR_ATTR_AFTER_EOF,
+  GUMBO_ERR_ATTR_AFTER_INVALID,
+  GUMBO_ERR_DUPLICATE_ATTR,
+  GUMBO_ERR_SOLIDUS_EOF,
+  GUMBO_ERR_SOLIDUS_INVALID,
+  GUMBO_ERR_DASHES_OR_DOCTYPE,
+  GUMBO_ERR_COMMENT_EOF,
+  GUMBO_ERR_COMMENT_INVALID,
+  GUMBO_ERR_COMMENT_BANG_AFTER_DOUBLE_DASH,
+  GUMBO_ERR_COMMENT_DASH_AFTER_DOUBLE_DASH,
+  GUMBO_ERR_COMMENT_SPACE_AFTER_DOUBLE_DASH,
+  GUMBO_ERR_COMMENT_END_BANG_EOF,
+  GUMBO_ERR_DOCTYPE_EOF,
+  GUMBO_ERR_DOCTYPE_INVALID,
+  GUMBO_ERR_DOCTYPE_SPACE,
+  GUMBO_ERR_DOCTYPE_RIGHT_BRACKET,
+  GUMBO_ERR_DOCTYPE_SPACE_OR_RIGHT_BRACKET,
+  GUMBO_ERR_DOCTYPE_END,
+  GUMBO_ERR_PARSER,
+  GUMBO_ERR_UNACKNOWLEDGED_SELF_CLOSING_TAG,
+} GumboErrorType;
+
+// Additional data for duplicated attributes.
+typedef struct _GumboDuplicateAttrError {
+  // The name of the attribute.  Owned by this struct.
+  const char* name;
+
+  // The (0-based) index within the attributes vector of the original
+  // occurrence.
+  unsigned int original_index;
+
+  // The (0-based) index where the new occurrence would be.
+  unsigned int new_index;
+} GumboDuplicateAttrError;
+
+// A simplified representation of the tokenizer state, designed to be more
+// useful to clients of this library than the internal representation.  This
+// condenses the actual states used in the tokenizer state machine into a few
+// values that will be familiar to users of HTML.
+typedef enum {
+  GUMBO_ERR_TOKENIZER_DATA,
+  GUMBO_ERR_TOKENIZER_CHAR_REF,
+  GUMBO_ERR_TOKENIZER_RCDATA,
+  GUMBO_ERR_TOKENIZER_RAWTEXT,
+  GUMBO_ERR_TOKENIZER_PLAINTEXT,
+  GUMBO_ERR_TOKENIZER_SCRIPT,
+  GUMBO_ERR_TOKENIZER_TAG,
+  GUMBO_ERR_TOKENIZER_SELF_CLOSING_TAG,
+  GUMBO_ERR_TOKENIZER_ATTR_NAME,
+  GUMBO_ERR_TOKENIZER_ATTR_VALUE,
+  GUMBO_ERR_TOKENIZER_MARKUP_DECLARATION,
+  GUMBO_ERR_TOKENIZER_COMMENT,
+  GUMBO_ERR_TOKENIZER_DOCTYPE,
+  GUMBO_ERR_TOKENIZER_CDATA,
+} GumboTokenizerErrorState;
+
+// Additional data for tokenizer errors.
+// This records the current state and codepoint encountered - this is usually
+// enough to reconstruct what went wrong and provide a friendly error message.
+typedef struct _GumboTokenizerError {
+  // The bad codepoint encountered.
+  int codepoint;
+
+  // The state that the tokenizer was in at the time.
+  GumboTokenizerErrorState state;
+} GumboTokenizerError;
+
+// Additional data for parse errors.
+typedef struct _GumboParserError {
+  // The type of input token that resulted in this error.
+  GumboTokenType input_type;
+
+  // The HTML tag of the input token.  TAG_UNKNOWN if this was not a tag token.
+  GumboTag input_tag;
+
+  // The insertion mode that the parser was in at the time.
+  GumboInsertionMode parser_state;
+
+  // The tag stack at the point of the error.  Note that this is an GumboVector
+  // of GumboTag's *stored by value* - cast the void* to an GumboTag directly to
+  // get at the tag.
+  GumboVector /* GumboTag */ tag_stack;
+} GumboParserError;
+
+// The overall error struct representing an error in decoding/tokenizing/parsing
+// the HTML.  This contains an enumerated type flag, a source position, and then
+// a union of fields containing data specific to the error.
+typedef struct _GumboError {
+  // The type of error.
+  GumboErrorType type;
+
+  // The position within the source file where the error occurred.
+  GumboSourcePosition position;
+
+  // A pointer to the byte within the original source file text where the error
+  // occurred (note that this is not the same as position.offset, as that gives
+  // character-based instead of byte-based offsets).
+  const char* original_text;
+
+  // Type-specific error information.
+  union {
+    // The code point we encountered, for:
+    // * GUMBO_ERR_UTF8_INVALID
+    // * GUMBO_ERR_UTF8_TRUNCATED
+    // * GUMBO_ERR_NUMERIC_CHAR_REF_WITHOUT_SEMICOLON
+    // * GUMBO_ERR_NUMERIC_CHAR_REF_INVALID
+    uint64_t codepoint;
+
+    // Tokenizer errors.
+    GumboTokenizerError tokenizer;
+
+    // Short textual data, for:
+    // * GUMBO_ERR_NAMED_CHAR_REF_WITHOUT_SEMICOLON
+    // * GUMBO_ERR_NAMED_CHAR_REF_INVALID
+    GumboStringPiece text;
+
+    // Duplicate attribute data, for GUMBO_ERR_DUPLICATE_ATTR.
+    GumboDuplicateAttrError duplicate_attr;
+
+    // Parser state, for GUMBO_ERR_PARSER and
+    // GUMBO_ERR_UNACKNOWLEDGE_SELF_CLOSING_TAG.
+    struct _GumboParserError parser;
+  } v;
+} GumboError;
+
+// Adds a new error to the parser's error list, and returns a pointer to it so
+// that clients can fill out the rest of its fields.  May return NULL if we're
+// already over the max_errors field specified in GumboOptions.
+GumboError* gumbo_add_error(struct _GumboParser* parser);
+
+// Initializes the errors vector in the parser.
+void gumbo_init_errors(struct _GumboParser* errors);
+
+// Frees all the errors in the 'errors_' field of the parser.
+void gumbo_destroy_errors(struct _GumboParser* errors);
+
+// Frees the memory used for a single GumboError.
+void gumbo_error_destroy(struct _GumboParser* parser, GumboError* error);
+
+// Prints an error to a string.  This fills an empty GumboStringBuffer with a
+// freshly-allocated buffer containing the error message text.  The caller is
+// responsible for deleting the buffer.  (Note that the buffer is allocated with
+// the allocator specified in the GumboParser config and hence should be freed
+// by gumbo_parser_deallocate().)
+void gumbo_error_to_string(
+    struct _GumboParser* parser, const GumboError* error,
+    GumboStringBuffer* output);
+
+// Prints a caret diagnostic to a string.  This fills an empty GumboStringBuffer
+// with a freshly-allocated buffer containing the error message text.  The
+// caller is responsible for deleting the buffer.  (Note that the buffer is
+// allocated with the allocator specified in the GumboParser config and hence
+// should be freed by gumbo_parser_deallocate().)
+void gumbo_caret_diagnostic_to_string(
+    struct _GumboParser* parser, const GumboError* error,
+    const char* source_text, GumboStringBuffer* output);
+
+// Like gumbo_caret_diagnostic_to_string, but prints the text to stdout instead
+// of writing to a string.
+void gumbo_print_caret_diagnostic(
+    struct _GumboParser* parser, const GumboError* error,
+    const char* source_text);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_ERROR_H_