nokogumbo 0.3 → 0.4

data/work/utf8.h

@@ -0,0 +1,127 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// This contains an implementation of a UTF8 iterator and decoder suitable for
+// an HTML5 parser.  This does a bit more than straight UTF-8 decoding.  The
+// HTML5 spec specifies that:
+// 1. Decoding errors are parse errors.
+// 2. Certain other codepoints (eg. control characters) are parse errors.
+// 3. Carriage returns and CR/LF groups are converted to line feeds.
+// http://www.whatwg.org/specs/web-apps/current-work/multipage/infrastructure.html#decoded-as-utf-8,-with-error-handling
+//
+// Also, we want to keep track of source positions for error handling.  As a
+// result, we fold all that functionality into this decoder, and can't use an
+// off-the-shelf library.
+//
+// This header is internal-only, which is why we prefix functions with only
+// utf8_ or utf8_iterator_ instead of gumbo_utf8_.
+
+#ifndef GUMBO_UTF8_H_
+#define GUMBO_UTF8_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "gumbo.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct _GumboError;
+struct _GumboParser;
+
+// Unicode replacement char.
+extern const int kUtf8ReplacementChar;
+
+typedef struct _Utf8Iterator {
+  // Points at the start of the code point most recently read into 'current'.
+  const char* _start;
+
+  // Points at the mark.  The mark is initially set to the beginning of the
+  // input.
+  const char* _mark;
+
+  // Points past the end of the iter, like a past-the-end iterator in the STL.
+  const char* _end;
+
+  // The code point under the cursor.
+  int _current;
+
+  // The width in bytes of the current code point.
+  int _width;
+
+  // The SourcePosition for the current location.
+  GumboSourcePosition _pos;
+
+  // The SourcePosition for the mark.
+  GumboSourcePosition _mark_pos;
+
+  // Pointer back to the GumboParser instance, for configuration options and
+  // error recording.
+  struct _GumboParser* _parser;
+} Utf8Iterator;
+
+// Returns true if this Unicode code point is in the list of characters
+// forbidden by the HTML5 spec, such as NUL bytes and undefined control chars.
+bool utf8_is_invalid_code_point(int c);
+
+// Initializes a new Utf8Iterator from the given byte buffer.  The source does
+// not have to be NUL-terminated, but the length must be passed in explicitly.
+void utf8iterator_init(
+    struct _GumboParser* parser, const char* source, size_t source_length,
+    Utf8Iterator* iter);
+
+// Advances the current position by one code point.
+void utf8iterator_next(Utf8Iterator* iter);
+
+// Returns the current code point as an integer.
+int utf8iterator_current(const Utf8Iterator* iter);
+
+// Retrieves and fills the output parameter with the current source position.
+void utf8iterator_get_position(
+    const Utf8Iterator* iter, GumboSourcePosition* output);
+
+// Retrieves a character pointer to the start of the current character.
+const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter);
+
+// If the upcoming text in the buffer matches the specified prefix (which has
+// length 'length'), consume it and return true.  Otherwise, return false with
+// no other effects.  If the length of the string would overflow the buffer,
+// this returns false.  Note that prefix should not contain null bytes because
+// of the use of strncmp/strncasecmp internally.  All existing use-cases adhere
+// to this.
+bool utf8iterator_maybe_consume_match(
+    Utf8Iterator* iter, const char* prefix, size_t length, bool case_sensitive);
+
+// "Marks" a particular location of interest in the input stream, so that it can
+// later be reset() to.  There's also the ability to record an error at the
+// point that was marked, as oftentimes that's more useful than the last
+// character before the error was detected.
+void utf8iterator_mark(Utf8Iterator* iter);
+
+// Returns the current input stream position to the mark.
+void utf8iterator_reset(Utf8Iterator* iter);
+
+// Sets the position and original text fields of an error to the value at the
+// mark.
+void utf8iterator_fill_error_at_mark(
+    Utf8Iterator* iter, struct _GumboError* error);
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // GUMBO_UTF8_H_

data/work/util.c

@@ -0,0 +1,58 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "util.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "gumbo.h"
+#include "parser.h"
+
+// TODO(jdtang): This should be elsewhere, but there's no .c file for
+// SourcePositions and yet the constant needs some linkage, so this is as good
+// as any.
+const GumboSourcePosition kGumboEmptySourcePosition = { 0, 0, 0 };
+
+void* gumbo_parser_allocate(GumboParser* parser, size_t num_bytes) {
+  return parser->_options->allocator(parser->_options->userdata, num_bytes);
+}
+
+void gumbo_parser_deallocate(GumboParser* parser, void* ptr) {
+  return parser->_options->deallocator(parser->_options->userdata, ptr);
+}
+
+char* gumbo_copy_stringz(GumboParser* parser, const char* str) {
+  char* buffer = gumbo_parser_allocate(parser, strlen(str) + 1);
+  strcpy(buffer, str);
+  return buffer;
+}
+
+// Debug function to trace operation of the parser.  Pass --copts=-DGUMBO_DEBUG
+// to use.
+void gumbo_debug(const char* format, ...) {
+#ifdef GUMBO_DEBUG
+  va_list args;
+  va_start(args, format);
+  vprintf(format, args);
+  va_end(args);
+  fflush(stdout);
+#endif
+}

data/work/util.h

@@ -0,0 +1,57 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+//
+// This contains some utility functions that didn't fit into any of the other
+// headers.
+
+#ifndef GUMBO_UTIL_H_
+#define GUMBO_UTIL_H_
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Forward declaration since it's passed into some of the functions in this
+// header.
+struct _GumboParser;
+
+// Utility function for allocating & copying a null-terminated string into a
+// freshly-allocated buffer.  This is necessary for proper memory management; we
+// have the convention that all const char* in parse tree structures are
+// freshly-allocated, so if we didn't copy, we'd try to delete a literal string
+// when the parse tree is destroyed.
+char* gumbo_copy_stringz(struct _GumboParser* parser, const char* str);
+
+// Allocate a chunk of memory, using the allocator specified in the Parser's
+// config options.
+void* gumbo_parser_allocate(struct _GumboParser* parser, size_t num_bytes);
+
+// Deallocate a chunk of memory, using the deallocator specified in the Parser's
+// config options.
+void gumbo_parser_deallocate(struct _GumboParser* parser, void* ptr);
+
+// Debug wrapper for printf, to make it easier to turn off debugging info when
+// required.
+void gumbo_debug(const char* format, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_UTIL_H_

data/work/vector.c

@@ -0,0 +1,121 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#include "vector.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include "util.h"
+
+struct _GumboParser;
+
+const GumboVector kGumboEmptyVector = { NULL, 0, 0 };
+
+void gumbo_vector_init(
+    struct _GumboParser* parser, size_t initial_capacity, GumboVector* vector) {
+  vector->length = 0;
+  vector->capacity = initial_capacity;
+  if (initial_capacity > 0) {
+    vector->data = gumbo_parser_allocate(
+        parser, sizeof(void*) * initial_capacity);
+  } else {
+    vector->data = NULL;
+  }
+}
+
+void gumbo_vector_destroy(struct _GumboParser* parser, GumboVector* vector) {
+  if (vector->capacity > 0) {
+    gumbo_parser_deallocate(parser, vector->data);
+  }
+}
+
+static void enlarge_vector_if_full(
+    struct _GumboParser* parser, GumboVector* vector) {
+  if (vector->length >= vector->capacity) {
+    if (vector->capacity) {
+      size_t old_num_bytes = sizeof(void*) * vector->capacity;
+      vector->capacity *= 2;
+      size_t num_bytes = sizeof(void*) * vector->capacity;
+      void** temp = gumbo_parser_allocate(parser, num_bytes);
+      memcpy(temp, vector->data, old_num_bytes);
+      gumbo_parser_deallocate(parser, vector->data);
+      vector->data = temp;
+    } else {
+      // 0-capacity vector; no previous array to deallocate.
+      vector->capacity = 2;
+      vector->data = gumbo_parser_allocate(
+          parser, sizeof(void*) * vector->capacity);
+    }
+  }
+}
+
+void gumbo_vector_add(
+    struct _GumboParser* parser, void* element, GumboVector* vector) {
+  enlarge_vector_if_full(parser, vector);
+  assert(vector->data);
+  assert(vector->length < vector->capacity);
+  vector->data[vector->length++] = element;
+}
+
+void* gumbo_vector_pop(struct _GumboParser* parser, GumboVector* vector) {
+  if (vector->length == 0) {
+    return NULL;
+  }
+  return vector->data[--vector->length];
+}
+
+int gumbo_vector_index_of(GumboVector* vector, void* element) {
+  for (int i = 0; i < vector->length; ++i) {
+    if (vector->data[i] == element) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+void gumbo_vector_insert_at(
+    struct _GumboParser* parser, void* element, int index, GumboVector* vector) {
+  assert(index >= 0);
+  assert(index <= vector->length);
+  enlarge_vector_if_full(parser, vector);
+  ++vector->length;
+  memmove(&vector->data[index + 1], &vector->data[index],
+          sizeof(void*) * (vector->length - index - 1));
+  vector->data[index] = element;
+}
+
+void gumbo_vector_remove(
+    struct _GumboParser* parser, void* node, GumboVector* vector) {
+  int index = gumbo_vector_index_of(vector, node);
+  if (index == -1) {
+    return;
+  }
+  gumbo_vector_remove_at(parser, index, vector);
+}
+
+void* gumbo_vector_remove_at(
+    struct _GumboParser* parser, int index, GumboVector* vector) {
+  assert(index >= 0);
+  assert(index < vector->length);
+  void* result = vector->data[index];
+  memmove(&vector->data[index], &vector->data[index + 1],
+          sizeof(void*) * (vector->length - index - 1));
+  --vector->length;
+  return result;
+}

data/work/vector.h

@@ -0,0 +1,66 @@
+// Copyright 2010 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jdtang@google.com (Jonathan Tang)
+
+#ifndef GUMBO_VECTOR_H_
+#define GUMBO_VECTOR_H_
+
+#include "gumbo.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Forward declaration since it's passed into some of the functions in this
+// header.
+struct _GumboParser;
+
+// Initializes a new GumboVector with the specified initial capacity.
+void gumbo_vector_init(
+    struct _GumboParser* parser, size_t initial_capacity, GumboVector* vector);
+
+// Frees the memory used by an GumboVector.  Does not free the contained
+// pointers.
+void gumbo_vector_destroy(struct _GumboParser* parser, GumboVector* vector);
+
+// Adds a new element to an GumboVector.
+void gumbo_vector_add(
+    struct _GumboParser* parser, void* element, GumboVector* vector);
+
+// Removes and returns the element most recently added to the GumboVector.
+// Ownership is transferred to caller.  Capacity is unchanged.  If the vector is
+// empty, NULL is returned.
+void* gumbo_vector_pop(struct _GumboParser* parser, GumboVector* vector);
+
+// Inserts an element at a specific index.  This is potentially O(N) time, but
+// is necessary for some of the spec's behavior.
+void gumbo_vector_insert_at(
+    struct _GumboParser* parser, void* element, int index, GumboVector* vector);
+
+// Removes an element from the vector, or does nothing if the element is not in
+// the vector.
+void gumbo_vector_remove(
+    struct _GumboParser* parser, void* element, GumboVector* vector);
+
+// Removes and returns an element at a specific index.  Note that this is
+// potentially O(N) time and should be used sparingly.
+void* gumbo_vector_remove_at(
+    struct _GumboParser* parser, int index, GumboVector* vector);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // GUMBO_VECTOR_H_

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nokogumbo
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: '0.4'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-19 00:00:00.000000000 Z
+date: 2013-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -35,11 +35,37 @@ extensions:
 - work/extconf.rb
 extra_rdoc_files: []
 files:
+- work/extconf.rb
+- work/nokogumbo.c
+- work/vector.c
+- work/string_buffer.c
+- work/error.c
+- work/attribute.c
+- work/char_ref.c
+- work/string_piece.c
+- work/tag.c
+- work/util.c
+- work/parser.c
+- work/utf8.c
+- work/tokenizer.c
+- work/utf8.h
+- work/token_type.h
+- work/util.h
+- work/tokenizer_states.h
+- work/error.h
+- work/parser.h
+- work/tokenizer.h
+- work/string_buffer.h
+- work/vector.h
+- work/string_piece.h
+- work/gumbo.h
+- work/attribute.h
+- work/char_ref.h
+- work/insertion_mode.h
 - lib/nokogumbo.rb
 - LICENSE.txt
 - Rakefile
 - README.md
-- work/extconf.rb
 homepage: https://github.com/rubys/nokogumbo/#readme
 licenses:
 - Apache 2.0