nokogumbo 1.4.8 → 1.4.9

data/gumbo-parser/src/parser.h

@@ -0,0 +1,57 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// Contains the definition of the top-level GumboParser structure that's
+// threaded through basically every internal function in the library.
+
+#ifndef GUMBO_PARSER_H_
+#define GUMBO_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct GumboInternalParserState;
+struct GumboInternalOutput;
+struct GumboInternalOptions;
+struct GumboInternalTokenizerState;
+
+// An overarching struct that's threaded through (nearly) all functions in the
+// library, OOP-style.  This gives each function access to the options and
+// output, along with any internal state needed for the parse.
+typedef struct GumboInternalParser {
+  // Settings for this parse run.
+  const struct GumboInternalOptions* _options;
+
+  // Output for the parse.
+  struct GumboInternalOutput* _output;
+
+  // The internal tokenizer state, defined as a pointer to avoid a cyclic
+  // dependency on html5tokenizer.h.  The main parse routine is responsible for
+  // initializing this on parse start, and destroying it on parse end.
+  // End-users will never see a non-garbage value in this pointer.
+  struct GumboInternalTokenizerState* _tokenizer_state;
+
+  // The internal parser state.  Initialized on parse start and destroyed on
+  // parse end; end-users will never see a non-garbage value in this pointer.
+  struct GumboInternalParserState* _parser_state;
+} GumboParser;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_PARSER_H_

data/gumbo-parser/src/string_buffer.c

@@ -0,0 +1,110 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "string_buffer.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include "string_piece.h"
+#include "util.h"
+
+struct GumboInternalParser;
+
+// Size chosen via statistical analysis of ~60K websites.
+// 99% of text nodes and 98% of attribute names/values fit in this initial size.
+static const size_t kDefaultStringBufferSize = 5;
+
+static void maybe_resize_string_buffer(struct GumboInternalParser* parser,
+    size_t additional_chars, GumboStringBuffer* buffer) {
+  size_t new_length = buffer->length + additional_chars;
+  size_t new_capacity = buffer->capacity;
+  while (new_capacity < new_length) {
+    new_capacity *= 2;
+  }
+  if (new_capacity != buffer->capacity) {
+    char* new_data = gumbo_parser_allocate(parser, new_capacity);
+    memcpy(new_data, buffer->data, buffer->length);
+    gumbo_parser_deallocate(parser, buffer->data);
+    buffer->data = new_data;
+    buffer->capacity = new_capacity;
+  }
+}
+
+void gumbo_string_buffer_init(
+    struct GumboInternalParser* parser, GumboStringBuffer* output) {
+  output->data = gumbo_parser_allocate(parser, kDefaultStringBufferSize);
+  output->length = 0;
+  output->capacity = kDefaultStringBufferSize;
+}
+
+void gumbo_string_buffer_reserve(struct GumboInternalParser* parser,
+    size_t min_capacity, GumboStringBuffer* output) {
+  maybe_resize_string_buffer(parser, min_capacity - output->length, output);
+}
+
+void gumbo_string_buffer_append_codepoint(
+    struct GumboInternalParser* parser, int c, GumboStringBuffer* output) {
+  // num_bytes is actually the number of continuation bytes, 1 less than the
+  // total number of bytes.  This is done to keep the loop below simple and
+  // should probably change if we unroll it.
+  int num_bytes, prefix;
+  if (c <= 0x7f) {
+    num_bytes = 0;
+    prefix = 0;
+  } else if (c <= 0x7ff) {
+    num_bytes = 1;
+    prefix = 0xc0;
+  } else if (c <= 0xffff) {
+    num_bytes = 2;
+    prefix = 0xe0;
+  } else {
+    num_bytes = 3;
+    prefix = 0xf0;
+  }
+  maybe_resize_string_buffer(parser, num_bytes + 1, output);
+  output->data[output->length++] = prefix | (c >> (num_bytes * 6));
+  for (int i = num_bytes - 1; i >= 0; --i) {
+    output->data[output->length++] = 0x80 | (0x3f & (c >> (i * 6)));
+  }
+}
+
+void gumbo_string_buffer_append_string(struct GumboInternalParser* parser,
+    GumboStringPiece* str, GumboStringBuffer* output) {
+  maybe_resize_string_buffer(parser, str->length, output);
+  memcpy(output->data + output->length, str->data, str->length);
+  output->length += str->length;
+}
+
+char* gumbo_string_buffer_to_string(
+    struct GumboInternalParser* parser, GumboStringBuffer* input) {
+  char* buffer = gumbo_parser_allocate(parser, input->length + 1);
+  memcpy(buffer, input->data, input->length);
+  buffer[input->length] = '\0';
+  return buffer;
+}
+
+void gumbo_string_buffer_clear(
+    struct GumboInternalParser* parser, GumboStringBuffer* input) {
+  input->length = 0;
+}
+
+void gumbo_string_buffer_destroy(
+    struct GumboInternalParser* parser, GumboStringBuffer* buffer) {
+  gumbo_parser_deallocate(parser, buffer->data);
+}

data/gumbo-parser/src/string_buffer.h

@@ -0,0 +1,84 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+#ifndef GUMBO_STRING_BUFFER_H_
+#define GUMBO_STRING_BUFFER_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "gumbo.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct GumboInternalParser;
+
+// A struct representing a mutable, growable string.  This consists of a
+// heap-allocated buffer that may grow (by doubling) as necessary.  When
+// converting to a string, this allocates a new buffer that is only as long as
+// it needs to be.  Note that the internal buffer here is *not* nul-terminated,
+// so be sure not to use ordinary string manipulation functions on it.
+typedef struct {
+  // A pointer to the beginning of the string.  NULL iff length == 0.
+  char* data;
+
+  // The length of the string fragment, in bytes.  May be zero.
+  size_t length;
+
+  // The capacity of the buffer, in bytes.
+  size_t capacity;
+} GumboStringBuffer;
+
+// Initializes a new GumboStringBuffer.
+void gumbo_string_buffer_init(
+    struct GumboInternalParser* parser, GumboStringBuffer* output);
+
+// Ensures that the buffer contains at least a certain amount of space.  Most
+// useful with snprintf and the other length-delimited string functions, which
+// may want to write directly into the buffer.
+void gumbo_string_buffer_reserve(struct GumboInternalParser* parser,
+    size_t min_capacity, GumboStringBuffer* output);
+
+// Appends a single Unicode codepoint onto the end of the GumboStringBuffer.
+// This is essentially a UTF-8 encoder, and may add 1-4 bytes depending on the
+// value of the codepoint.
+void gumbo_string_buffer_append_codepoint(
+    struct GumboInternalParser* parser, int c, GumboStringBuffer* output);
+
+// Appends a string onto the end of the GumboStringBuffer.
+void gumbo_string_buffer_append_string(struct GumboInternalParser* parser,
+    GumboStringPiece* str, GumboStringBuffer* output);
+
+// Converts this string buffer to const char*, alloctaing a new buffer for it.
+char* gumbo_string_buffer_to_string(
+    struct GumboInternalParser* parser, GumboStringBuffer* input);
+
+// Reinitialize this string buffer.  This clears it by setting length=0.  It
+// does not zero out the buffer itself.
+void gumbo_string_buffer_clear(
+    struct GumboInternalParser* parser, GumboStringBuffer* input);
+
+// Deallocates this GumboStringBuffer.
+void gumbo_string_buffer_destroy(
+    struct GumboInternalParser* parser, GumboStringBuffer* buffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_STRING_BUFFER_H_

data/gumbo-parser/src/string_piece.c

@@ -0,0 +1,48 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "string_piece.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include "util.h"
+
+struct GumboInternalParser;
+
+const GumboStringPiece kGumboEmptyString = {NULL, 0};
+
+bool gumbo_string_equals(
+    const GumboStringPiece* str1, const GumboStringPiece* str2) {
+  return str1->length == str2->length &&
+         !memcmp(str1->data, str2->data, str1->length);
+}
+
+bool gumbo_string_equals_ignore_case(
+    const GumboStringPiece* str1, const GumboStringPiece* str2) {
+  return str1->length == str2->length &&
+         !strncasecmp(str1->data, str2->data, str1->length);
+}
+
+void gumbo_string_copy(struct GumboInternalParser* parser,
+    GumboStringPiece* dest, const GumboStringPiece* source) {
+  dest->length = source->length;
+  char* buffer = gumbo_parser_allocate(parser, source->length);
+  memcpy(buffer, source->data, source->length);
+  dest->data = buffer;
+}

data/gumbo-parser/src/string_piece.h

@@ -0,0 +1,38 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#ifndef GUMBO_STRING_PIECE_H_
+#define GUMBO_STRING_PIECE_H_
+
+#include "gumbo.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct GumboInternalParser;
+
+// Performs a deep-copy of an GumboStringPiece, allocating a fresh buffer in the
+// destination and copying over the characters from source.  Dest should be
+// empty, with no buffer allocated; otherwise, this leaks it.
+void gumbo_string_copy(struct GumboInternalParser* parser,
+    GumboStringPiece* dest, const GumboStringPiece* source);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_STRING_PIECE_H_

data/gumbo-parser/src/tag.c

@@ -0,0 +1,95 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "gumbo.h"
+
+#include <assert.h>
+#include <ctype.h>
+#include <string.h>
+
+const char* kGumboTagNames[] = {
+#include "tag_strings.h"
+    "",  // TAG_UNKNOWN
+    "",  // TAG_LAST
+};
+
+static const unsigned char kGumboTagSizes[] = {
+#include "tag_sizes.h"
+    0,  // TAG_UNKNOWN
+    0,  // TAG_LAST
+};
+
+const char* gumbo_normalized_tagname(GumboTag tag) {
+  assert(tag <= GUMBO_TAG_LAST);
+  return kGumboTagNames[tag];
+}
+
+void gumbo_tag_from_original_text(GumboStringPiece* text) {
+  if (text->data == NULL) {
+    return;
+  }
+
+  assert(text->length >= 2);
+  assert(text->data[0] == '<');
+  assert(text->data[text->length - 1] == '>');
+  if (text->data[1] == '/') {
+    // End tag.
+    assert(text->length >= 3);
+    text->data += 2;  // Move past </
+    text->length -= 3;
+  } else {
+    // Start tag.
+    text->data += 1;  // Move past <
+    text->length -= 2;
+    // strnchr is apparently not a standard C library function, so I loop
+    // explicitly looking for whitespace or other illegal tag characters.
+    for (const char* c = text->data; c != text->data + text->length; ++c) {
+      if (isspace(*c) || *c == '/') {
+        text->length = c - text->data;
+        break;
+      }
+    }
+  }
+}
+
+static int case_memcmp(const char* s1, const char* s2, unsigned int n) {
+  while (n--) {
+    unsigned char c1 = tolower(*s1++);
+    unsigned char c2 = tolower(*s2++);
+    if (c1 != c2) return (int) c1 - (int) c2;
+  }
+  return 0;
+}
+
+#include "tag_gperf.h"
+#define TAG_MAP_SIZE (sizeof(kGumboTagMap) / sizeof(kGumboTagMap[0]))
+
+GumboTag gumbo_tagn_enum(const char* tagname, unsigned int length) {
+  if (length) {
+    unsigned int key = tag_hash(tagname, length);
+    if (key < TAG_MAP_SIZE) {
+      GumboTag tag = kGumboTagMap[key];
+      if (length == kGumboTagSizes[(int) tag] &&
+          !case_memcmp(tagname, kGumboTagNames[(int) tag], length))
+        return tag;
+    }
+  }
+  return GUMBO_TAG_UNKNOWN;
+}
+
+GumboTag gumbo_tag_enum(const char* tagname) {
+  return gumbo_tagn_enum(tagname, strlen(tagname));
+}

data/gumbo-parser/src/tag.in

@@ -0,0 +1,150 @@
+html
+head
+title
+base
+link
+meta
+style
+script
+noscript
+template
+body
+article
+section
+nav
+aside
+h1
+h2
+h3
+h4
+h5
+h6
+hgroup
+header
+footer
+address
+p
+hr
+pre
+blockquote
+ol
+ul
+li
+dl
+dt
+dd
+figure
+figcaption
+main
+div
+a
+em
+strong
+small
+s
+cite
+q
+dfn
+abbr
+data
+time
+code
+var
+samp
+kbd
+sub
+sup
+i
+b
+u
+mark
+ruby
+rt
+rp
+bdi
+bdo
+span
+br
+wbr
+ins
+del
+image
+img
+iframe
+embed
+object
+param
+video
+audio
+source
+track
+canvas
+map
+area
+math
+mi
+mo
+mn
+ms
+mtext
+mglyph
+malignmark
+annotation-xml
+svg
+foreignobject
+desc
+table
+caption
+colgroup
+col
+tbody
+thead
+tfoot
+tr
+td
+th
+form
+fieldset
+legend
+label
+input
+button
+select
+datalist
+optgroup
+option
+textarea
+keygen
+output
+progress
+meter
+details
+summary
+menu
+menuitem
+applet
+acronym
+bgsound
+dir
+frame
+frameset
+noframes
+isindex
+listing
+xmp
+nextid
+noembed
+plaintext
+rb
+strike
+basefont
+big
+blink
+center
+font
+marquee
+multicol
+nobr
+spacer
+tt
+rtc