nokogumbo 0.5 → 0.5.1

data/work/parser.h

@@ -0,0 +1,57 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// Contains the definition of the top-level GumboParser structure that's
+// threaded through basically every internal function in the library.
+
+#ifndef GUMBO_PARSER_H_
+#define GUMBO_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct _GumboParserState;
+struct _GumboOutput;
+struct _GumboOptions;
+struct _GumboTokenizerState;
+
+// An overarching struct that's threaded through (nearly) all functions in the
+// library, OOP-style.  This gives each function access to the options and
+// output, along with any internal state needed for the parse.
+typedef struct _GumboParser {
+  // Settings for this parse run.
+  const struct _GumboOptions* _options;
+
+  // Output for the parse.
+  struct _GumboOutput* _output;
+
+  // The internal tokenizer state, defined as a pointer to avoid a cyclic
+  // dependency on html5tokenizer.h.  The main parse routine is responsible for
+  // initializing this on parse start, and destroying it on parse end.
+  // End-users will never see a non-garbage value in this pointer.
+  struct _GumboTokenizerState* _tokenizer_state;
+
+  // The internal parser state.  Initialized on parse start and destroyed on
+  // parse end; end-users will never see a non-garbage value in this pointer.
+  struct _GumboParserState* _parser_state;
+} GumboParser;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_PARSER_H_

data/work/string_buffer.c

@@ -0,0 +1,106 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "string_buffer.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include "string_piece.h"
+#include "util.h"
+
+struct _GumboParser;
+
+static const size_t kDefaultStringBufferSize = 10;
+
+static void maybe_resize_string_buffer(
+    struct _GumboParser* parser, size_t additional_chars,
+    GumboStringBuffer* buffer) {
+  size_t new_length = buffer->length + additional_chars;
+  size_t new_capacity = buffer->capacity;
+  while (new_capacity < new_length) {
+    new_capacity *= 2;
+  }
+  if (new_capacity != buffer->capacity) {
+    char* new_data = gumbo_parser_allocate(parser, new_capacity);
+    memcpy(new_data, buffer->data, buffer->length);
+    gumbo_parser_deallocate(parser, buffer->data);
+    buffer->data = new_data;
+    buffer->capacity = new_capacity;
+  }
+}
+
+void gumbo_string_buffer_init(
+    struct _GumboParser* parser, GumboStringBuffer* output) {
+  output->data = gumbo_parser_allocate(parser, kDefaultStringBufferSize);
+  output->length = 0;
+  output->capacity = kDefaultStringBufferSize;
+}
+
+void gumbo_string_buffer_reserve(
+    struct _GumboParser* parser, size_t min_capacity,
+    GumboStringBuffer* output) {
+  maybe_resize_string_buffer(parser, min_capacity - output->length, output);
+}
+
+void gumbo_string_buffer_append_codepoint(
+    struct _GumboParser* parser, int c, GumboStringBuffer* output) {
+  // num_bytes is actually the number of continuation bytes, 1 less than the
+  // total number of bytes.  This is done to keep the loop below simple and
+  // should probably change if we unroll it.
+  int num_bytes, prefix;
+  if (c <= 0x7f) {
+    num_bytes = 0;
+    prefix = 0;
+  } else if (c <= 0x7ff) {
+    num_bytes = 1;
+    prefix = 0xc0;
+  } else if (c <= 0xffff) {
+    num_bytes = 2;
+    prefix = 0xe0;
+  } else {
+    num_bytes = 3;
+    prefix = 0xf0;
+  }
+  maybe_resize_string_buffer(parser, num_bytes + 1, output);
+  output->data[output->length++] = prefix | (c >> (num_bytes * 6));
+  for (int i = num_bytes - 1; i >= 0; --i) {
+    output->data[output->length++] = 0x80 | (0x3f & (c >> (i * 6)));
+  }
+}
+
+void gumbo_string_buffer_append_string(
+    struct _GumboParser* parser, GumboStringPiece* str,
+    GumboStringBuffer* output) {
+  maybe_resize_string_buffer(parser, str->length, output);
+  memcpy(output->data + output->length, str->data, str->length);
+  output->length += str->length;
+}
+
+char* gumbo_string_buffer_to_string(
+    struct _GumboParser* parser, GumboStringBuffer* input) {
+  char* buffer = gumbo_parser_allocate(parser, input->length + 1);
+  memcpy(buffer, input->data, input->length);
+  buffer[input->length] = '\0';
+  return buffer;
+}
+
+void gumbo_string_buffer_destroy(
+    struct _GumboParser* parser, GumboStringBuffer* buffer) {
+  gumbo_parser_deallocate(parser, buffer->data);
+}

data/work/string_buffer.h

@@ -0,0 +1,82 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+#ifndef GUMBO_STRING_BUFFER_H_
+#define GUMBO_STRING_BUFFER_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Forward declaration since it's passed into some of the functions in this
+// header.
+struct _GumboParser;
+struct _GumboStringPiece;
+
+// A struct representing a mutable, growable string.  This consists of a
+// heap-allocated buffer that may grow (by doubling) as necessary.  When
+// converting to a string, this allocates a new buffer that is only as long as
+// it needs to be.  Note that the internal buffer here is *not* nul-terminated,
+// so be sure not to use ordinary string manipulation functions on it.
+typedef struct _GumboStringBuffer {
+  // A pointer to the beginning of the string.  NULL iff length == 0.
+  char* data;
+
+  // The length of the string fragment, in bytes.  May be zero.
+  size_t length;
+
+  // The capacity of the buffer, in bytes.
+  size_t capacity;
+} GumboStringBuffer;
+
+// Initializes a new GumboStringBuffer.
+void gumbo_string_buffer_init(
+    struct _GumboParser* parser, GumboStringBuffer* output);
+
+// Ensures that the buffer contains at least a certain amount of space.  Most
+// useful with snprintf and the other length-delimited string functions, which
+// may want to write directly into the buffer.
+void gumbo_string_buffer_reserve(
+    struct _GumboParser* parser, size_t min_capacity,
+    GumboStringBuffer* output);
+
+// Appends a single Unicode codepoint onto the end of the GumboStringBuffer.
+// This is essentially a UTF-8 encoder, and may add 1-4 bytes depending on the
+// value of the codepoint.
+void gumbo_string_buffer_append_codepoint(
+    struct _GumboParser* parser, int c, GumboStringBuffer* output);
+
+// Appends a string onto the end of the GumboStringBuffer.
+void gumbo_string_buffer_append_string(
+    struct _GumboParser* parser, struct _GumboStringPiece* str,
+    GumboStringBuffer* output);
+
+// Converts this string buffer to const char*, alloctaing a new buffer for it.
+char* gumbo_string_buffer_to_string(
+    struct _GumboParser* parser, GumboStringBuffer* input);
+
+// Deallocates this GumboStringBuffer.
+void gumbo_string_buffer_destroy(
+    struct _GumboParser* parser, GumboStringBuffer* buffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_STRING_BUFFER_H_

data/work/string_piece.c

@@ -0,0 +1,49 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "string_piece.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include "util.h"
+
+struct _GumboParser;
+
+const GumboStringPiece kGumboEmptyString = { NULL, 0 };
+
+bool gumbo_string_equals(
+    const GumboStringPiece* str1, const GumboStringPiece* str2) {
+  return str1->length == str2->length &&
+      !memcmp(str1->data, str2->data, str1->length);
+}
+
+bool gumbo_string_equals_ignore_case(
+    const GumboStringPiece* str1, const GumboStringPiece* str2) {
+  return str1->length == str2->length &&
+      !strncasecmp(str1->data, str2->data, str1->length);
+}
+
+void gumbo_string_copy(
+    struct _GumboParser* parser, GumboStringPiece* dest,
+    const GumboStringPiece* source) {
+  dest->length = source->length;
+  char* buffer = gumbo_parser_allocate(parser, source->length);
+  memcpy(buffer, source->data, source->length);
+  dest->data = buffer;
+}

data/work/string_piece.h

@@ -0,0 +1,39 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#ifndef GUMBO_STRING_PIECE_H_
+#define GUMBO_STRING_PIECE_H_
+
+#include "gumbo.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct _GumboParser;
+
+// Performs a deep-copy of an GumboStringPiece, allocating a fresh buffer in the
+// destination and copying over the characters from source.  Dest should be
+// empty, with no buffer allocated; otherwise, this leaks it.
+void gumbo_string_copy(
+    struct _GumboParser* parser, GumboStringPiece* dest,
+    const GumboStringPiece* source);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_STRING_PIECE_H_

data/work/tag.c

@@ -0,0 +1,222 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "gumbo.h"
+
+#include <assert.h>
+#include <ctype.h>
+#include <strings.h>    // For strcasecmp.
+
+// NOTE(jdtang): Keep this in sync with the GumboTag enum in the header.
+// TODO(jdtang): Investigate whether there're efficiency benefits to putting the
+// most common tag names first, or to putting them in alphabetical order and
+// using a binary search.
+const char* kGumboTagNames[] = {
+  "html",
+  "head",
+  "title",
+  "base",
+  "link",
+  "meta",
+  "style",
+  "script",
+  "noscript",
+  "body",
+  "section",
+  "nav",
+  "article",
+  "aside",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "hgroup",
+  "header",
+  "footer",
+  "address",
+  "p",
+  "hr",
+  "pre",
+  "blockquote",
+  "ol",
+  "ul",
+  "li",
+  "dl",
+  "dt",
+  "dd",
+  "figure",
+  "figcaption",
+  "div",
+  "a",
+  "em",
+  "strong",
+  "small",
+  "s",
+  "cite",
+  "q",
+  "dfn",
+  "abbr",
+  "time",
+  "code",
+  "var",
+  "samp",
+  "kbd",
+  "sub",
+  "sup",
+  "i",
+  "b",
+  "mark",
+  "ruby",
+  "rt",
+  "rp",
+  "bdi",
+  "bdo",
+  "span",
+  "br",
+  "wbr",
+  "ins",
+  "del",
+  "image",
+  "img",
+  "iframe",
+  "embed",
+  "object",
+  "param",
+  "video",
+  "audio",
+  "source",
+  "track",
+  "canvas",
+  "map",
+  "area",
+  "math",
+  "mi",
+  "mo",
+  "mn",
+  "ms",
+  "mtext",
+  "mglyph",
+  "malignmark",
+  "annotation-xml",
+  "svg",
+  "foreignobject",
+  "desc",
+  "table",
+  "caption",
+  "colgroup",
+  "col",
+  "tbody",
+  "thead",
+  "tfoot",
+  "tr",
+  "td",
+  "th",
+  "form",
+  "fieldset",
+  "legend",
+  "label",
+  "input",
+  "button",
+  "select",
+  "datalist",
+  "optgroup",
+  "option",
+  "textarea",
+  "keygen",
+  "output",
+  "progress",
+  "meter",
+  "details",
+  "summary",
+  "command",
+  "menu",
+  "applet",
+  "acronym",
+  "bgsound",
+  "dir",
+  "frame",
+  "frameset",
+  "noframes",
+  "isindex",
+  "listing",
+  "xmp",
+  "nextid",
+  "noembed",
+  "plaintext",
+  "rb",
+  "strike",
+  "basefont",
+  "big",
+  "blink",
+  "center",
+  "font",
+  "marquee",
+  "multicol",
+  "nobr",
+  "spacer",
+  "tt",
+  "u",
+  "",                   // TAG_UNKNOWN
+  "",                   // TAG_LAST
+};
+
+const char* gumbo_normalized_tagname(GumboTag tag) {
+  assert(tag <= GUMBO_TAG_LAST);
+  return kGumboTagNames[tag];
+}
+
+// TODO(jdtang): Add test for this.
+void gumbo_tag_from_original_text(GumboStringPiece* text) {
+  if (text->data == NULL) {
+    return;
+  }
+
+  assert(text->length >= 2);
+  assert(text->data[0] == '<');
+  assert(text->data[text->length - 1] == '>');
+  if (text->data[1] == '/') {
+    // End tag.
+    assert(text->length >= 3);
+    text->data += 2;    // Move past </
+    text->length -= 3;
+  } else {
+    // Start tag.
+    text->data += 1;    // Move past <
+    text->length -= 2;
+    // strnchr is apparently not a standard C library function, so I loop
+    // explicitly looking for whitespace or other illegal tag characters.
+    for (const char* c = text->data; c != text->data + text->length; ++c) {
+      if (isspace(*c) || *c == '/') {
+        text->length = c - text->data;
+        break;
+      }
+    }
+  }
+}
+
+GumboTag gumbo_tag_enum(const char* tagname) {
+  for (int i = 0; i < GUMBO_TAG_LAST; ++i) {
+    // TODO(jdtang): strcasecmp is non-portable, so if we want to support
+    // non-GCC compilers, we'll need some #ifdef magic.  This source already has
+    // pretty significant issues with MSVC6 anyway.
+    if (strcasecmp(tagname, kGumboTagNames[i]) == 0) {
+      return i;
+    }
+  }
+  return GUMBO_TAG_UNKNOWN;
+}

data/work/token_type.h

@@ -0,0 +1,40 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#ifndef GUMBO_TOKEN_TYPE_H_
+#define GUMBO_TOKEN_TYPE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// An enum representing the type of token.
+typedef enum _GumboTokenType {
+  GUMBO_TOKEN_DOCTYPE,
+  GUMBO_TOKEN_START_TAG,
+  GUMBO_TOKEN_END_TAG,
+  GUMBO_TOKEN_COMMENT,
+  GUMBO_TOKEN_WHITESPACE,
+  GUMBO_TOKEN_CHARACTER,
+  GUMBO_TOKEN_NULL,
+  GUMBO_TOKEN_EOF
+} GumboTokenType;
+
+#ifdef __cplusplus
+}  // extern C
+#endif
+
+#endif  // GUMBO_TOKEN_TYPE_H_