html_tokenizer 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3d58539284af566692b81cc4633af1137baabea
-  data.tar.gz: 1877010598cbadadb27212eae39346769fa2afde
+  metadata.gz: 2e620e43f97a82c4cb3aae2067a2666325b453a5
+  data.tar.gz: 57784c1b53c4faefe2ab3b6e222836bfad852d8c
 SHA512:
-  metadata.gz: 9d6e46dfd48e7bb4967cfa13e90bdd213331b6dd38f9a57739ac1317f4b8147a9f11c5a39001ebf1c36554350ec444eaccc19dfe04040dadf2fc71d92435d5d5
-  data.tar.gz: f8fad88c25ff9404d710d4609ffc89fe80b08a71ba864cdca91a50cb9c5c98844befb75c43f5d4c6e0c56641fba57b4fda35ddf1c485abdcaf6bcd121bb4b0be
+  metadata.gz: 3b7a26cc219ea9f5885999146015e8137264ee51b43e5e6929d377cc2447ca6d4c1d5804e32954c24b2b807a19edcaf1f9cc708e7b5a0f83b086e16a21c8fa3e
+  data.tar.gz: 46beb0ed1994fe7468ab89451de1af830251da73d15c5d0943d30b70080806a458a7531a9fb7df2d7640e5ba1a195706eb84bf8896ef9184a207a1477747c200

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    html_tokenizer (0.0.1)
+    html_tokenizer (0.0.2)
 
 GEM
   remote: https://rubygems.org/
@@ -21,4 +21,4 @@ DEPENDENCIES
   rake-compiler
 
 BUNDLED WITH
-   1.12.3
+   1.16.0

data/ext/html_tokenizer_ext/parser.c

@@ -1,4 +1,5 @@
 #include <ruby.h>
+#include <ruby/encoding.h>
 #include "html_tokenizer.h"
 #include "parser.h"
 
@@ -65,6 +66,7 @@ static inline void parser_append_ref(struct token_reference_t *dest, struct toke
   if(dest->type == TOKEN_NONE || dest->type != src->type || (dest->start + dest->length) != src->start) {
     dest->type = src->type;
     dest->start = src->start;
+    dest->mb_start = src->mb_start;
     dest->length = src->length;
     dest->line_number = src->line_number;
     dest->column_number = src->column_number;
@@ -362,15 +364,21 @@ static inline int rawtext_context(struct parser_t *parser)
 
 static void parser_adjust_line_number(struct parser_t *parser, long unsigned int start, long unsigned int length)
 {
+  rb_encoding *enc = rb_enc_from_index(parser->doc.enc_index);
   long unsigned int i;
+  const char *buf, *nextlf;
 
-  for(i = start;i < (start + length); i++) {
-    if(parser->doc.data[i] == '\n') {
+  for(i = 0; i < length;) {
+    buf = &parser->doc.data[start + i];
+    nextlf = memchr(buf, '\n', length - i);
+    if(nextlf) {
       parser->doc.column_number = 0;
       parser->doc.line_number += 1;
+      i += (nextlf - buf) + 1;
     }
     else {
-      parser->doc.column_number += 1;
+      parser->doc.column_number += rb_enc_strlen(buf, buf + length - i, enc);
+      break;
     }
   }
 
@@ -383,11 +391,14 @@ static void parser_tokenize_callback(struct tokenizer_t *tk, enum token_type typ
   struct token_reference_t ref = {
     .type = type,
     .start = tk->scan.cursor,
+    .mb_start = tk->scan.mb_cursor,
     .length = length,
     .line_number = parser->doc.line_number,
     .column_number = parser->doc.column_number,
   };
   int parse_again = 1;
+  long unsigned int mb_strlen;
+  rb_encoding *enc;
 
   while(parse_again) {
     switch(parser->context)
@@ -438,8 +449,10 @@ static void parser_tokenize_callback(struct tokenizer_t *tk, enum token_type typ
   }
 
   if(rb_block_given_p()) {
+    enc = rb_enc_from_index(parser->doc.enc_index);
+    mb_strlen = rb_enc_strlen(parser->doc.data + ref.start, parser->doc.data + ref.start + ref.length, enc);
     rb_yield_values(5, token_type_to_symbol(type),
-      INT2NUM(ref.start), INT2NUM(ref.start + ref.length),
+      INT2NUM(ref.mb_start), INT2NUM(ref.mb_start + mb_strlen),
       INT2NUM(ref.line_number), INT2NUM(ref.column_number));
   }
 
@@ -465,6 +478,8 @@ static VALUE parser_initialize_method(VALUE self)
 
   parser->doc.length = 0;
   parser->doc.data = NULL;
+  parser->doc.enc_index = 0;
+  parser->doc.mb_length = 0;
 
   parser->doc.line_number = 1;
   parser->doc.column_number = 0;
@@ -478,11 +493,17 @@ static VALUE parser_initialize_method(VALUE self)
 static int parser_document_append(struct parser_t *parser, const char *string, unsigned long int length)
 {
   void *old = parser->doc.data;
+  unsigned long int mb_length;
+  char *buf;
+  rb_encoding *enc = rb_enc_from_index(parser->doc.enc_index);
   REALLOC_N(parser->doc.data, char, parser->doc.length + length + 1);
   DBG_PRINT("parser=%p realloc(parser->doc.data) %p -> %p length=%lu", parser, old,
-    parser->doc.data,  parser->doc.length + length + 1);
-  strcpy(parser->doc.data+parser->doc.length, string);
+    parser->doc.data, parser->doc.length + length + 1);
+  buf = parser->doc.data + parser->doc.length;
+  strcpy(buf, string);
+  mb_length = rb_enc_strlen(buf, buf + length, enc);
   parser->doc.length += length;
+  parser->doc.mb_length += mb_length;
   return 1;
 }
 
@@ -490,7 +511,7 @@ static VALUE parser_append_data(VALUE self, VALUE source, int is_placeholder)
 {
   struct parser_t *parser = NULL;
   char *string = NULL;
-  long unsigned int length = 0, cursor = 0;
+  long unsigned int length = 0, cursor = 0, mb_cursor = 0;
 
   if(NIL_P(source))
     return Qnil;
@@ -502,6 +523,15 @@ static VALUE parser_append_data(VALUE self, VALUE source, int is_placeholder)
   length = strlen(string);
 
   cursor = parser->doc.length;
+  mb_cursor = parser->doc.mb_length;
+
+  if(parser->doc.data == NULL) {
+    parser->doc.enc_index = rb_enc_get_index(source);
+  }
+  else if(parser->doc.enc_index != rb_enc_get_index(source)) {
+    rb_raise(rb_eArgError, "cannot append %s string to %s document",
+      rb_enc_name(rb_enc_get(source)), rb_enc_name(rb_enc_from_index(parser->doc.enc_index)));
+  }
 
   if(!parser_document_append(parser, string, length)) {
     // error
@@ -515,6 +545,8 @@ static VALUE parser_append_data(VALUE self, VALUE source, int is_placeholder)
     parser->tk.scan.cursor = cursor;
     parser->tk.scan.string = parser->doc.data;
     parser->tk.scan.length = parser->doc.length;
+    parser->tk.scan.enc_index = parser->doc.enc_index;
+    parser->tk.scan.mb_cursor = mb_cursor;
 
     tokenizer_scan_all(&parser->tk);
   }
@@ -535,17 +567,30 @@ static VALUE parser_append_placeholder_method(VALUE self, VALUE source)
 static VALUE parser_document_method(VALUE self)
 {
   struct parser_t *parser = NULL;
+  rb_encoding *enc;
   Parser_Get_Struct(self, parser);
   if(!parser->doc.data)
     return Qnil;
-  return rb_str_new(parser->doc.data, parser->doc.length);
+  enc = rb_enc_from_index(parser->doc.enc_index);
+  return rb_enc_str_new(parser->doc.data, parser->doc.length, enc);
 }
 
 static VALUE parser_document_length_method(VALUE self)
 {
   struct parser_t *parser = NULL;
+  rb_encoding *enc;
+  const char *buf;
+
   Parser_Get_Struct(self, parser);
-  return ULONG2NUM(parser->doc.length);
+
+  if(parser->doc.data == NULL) {
+    return ULONG2NUM(0);
+  }
+  else {
+    buf = parser->doc.data;
+    enc = rb_enc_from_index(parser->doc.enc_index);
+    return ULONG2NUM(rb_enc_strlen(buf, buf + parser->doc.length, enc));
+  }
 }
 
 static VALUE parser_context_method(VALUE self)
@@ -588,9 +633,10 @@ static VALUE parser_context_method(VALUE self)
 
 static inline VALUE ref_to_str(struct parser_t *parser, struct token_reference_t *ref)
 {
+  rb_encoding *enc = rb_enc_from_index(parser->doc.enc_index);
   if(ref->type == TOKEN_NONE || parser->doc.data == NULL)
     return Qnil;
-  return rb_str_new(parser->doc.data+ref->start, ref->length);
+  return rb_enc_str_new(parser->doc.data+ref->start, ref->length, enc);
 }
 
 static VALUE parser_tag_name_method(VALUE self)
@@ -665,29 +711,6 @@ static VALUE parser_rawtext_text_method(VALUE self)
   return ref_to_str(parser, &parser->rawtext.text);
 }
 
-static VALUE parser_extract_method(VALUE self, VALUE start_p, VALUE end_p)
-{
-  struct parser_t *parser = NULL;
-  unsigned long int start, end;
-  struct token_reference_t ref;
-
-  Parser_Get_Struct(self, parser);
-
-  start = NUM2ULONG(start_p);
-  end = NUM2ULONG(end_p);
-  if(end < start) {
-    rb_raise(rb_eArgError, "'end' must be greater or equal than 'start'");
-  }
-  if(end > parser->doc.length) {
-    rb_raise(rb_eArgError, "'end' argument not in range of document");
-  }
-
-  ref.type = TOKEN_TEXT; // anything not NONE
-  ref.start = start;
-  ref.length = end - start;
-  return ref_to_str(parser, &ref);
-}
-
 static VALUE parser_errors_count_method(VALUE self)
 {
   struct parser_t *parser = NULL;
@@ -749,7 +772,6 @@ void Init_html_tokenizer_parser(VALUE mHtmlTokenizer)
   rb_define_method(cParser, "column_number", parser_column_number_method, 0);
   rb_define_method(cParser, "parse", parser_parse_method, 1);
   rb_define_method(cParser, "append_placeholder", parser_append_placeholder_method, 1);
-  rb_define_method(cParser, "extract", parser_extract_method, 2);
   rb_define_method(cParser, "context", parser_context_method, 0);
   rb_define_method(cParser, "tag_name", parser_tag_name_method, 0);
   rb_define_method(cParser, "closing_tag?", parser_closing_tag_method, 0);

data/ext/html_tokenizer_ext/parser.h

@@ -28,11 +28,15 @@ struct parser_document_t {
   char *data;
   long unsigned int line_number;
   long unsigned int column_number;
+
+  int enc_index;
+  long unsigned int mb_length;
 };
 
 struct token_reference_t {
   enum token_type type;
   long unsigned int start;
+  long unsigned int mb_start;
   long unsigned int length;
   long unsigned int line_number;
   long unsigned int column_number;

data/ext/html_tokenizer_ext/tokenizer.c

@@ -1,4 +1,5 @@
 #include <ruby.h>
+#include <ruby/encoding.h>
 #include "html_tokenizer.h"
 #include "tokenizer.h"
 
@@ -60,6 +61,8 @@ void tokenizer_init(struct tokenizer_t *tk)
   tk->scan.string = NULL;
   tk->scan.cursor = 0;
   tk->scan.length = 0;
+  tk->scan.mb_cursor = 0;
+  tk->scan.enc_index = 0;
 
   tk->attribute_value_start = 0;
   tk->found_attribute = 0;
@@ -115,17 +118,27 @@ VALUE token_type_to_symbol(enum token_type type)
   return Qnil;
 }
 
+static long unsigned int tokenizer_mblength(struct tokenizer_t *tk, long unsigned int length)
+{
+  rb_encoding *enc = rb_enc_from_index(tk->scan.enc_index);
+  const char *buf = tk->scan.string + tk->scan.cursor;
+  return rb_enc_strlen(buf, buf + length, enc);
+}
+
 static void tokenizer_yield_tag(struct tokenizer_t *tk, enum token_type type, long unsigned int length, void *data)
 {
+  long unsigned int mb_length = tokenizer_mblength(tk, length);
   tk->last_token = type;
-  rb_yield_values(3, token_type_to_symbol(type), INT2NUM(tk->scan.cursor), INT2NUM(tk->scan.cursor + length));
+  rb_yield_values(3, token_type_to_symbol(type), INT2NUM(tk->scan.mb_cursor), INT2NUM(tk->scan.mb_cursor + mb_length));
 }
 
 static void tokenizer_callback(struct tokenizer_t *tk, enum token_type type, long unsigned int length)
 {
+  long unsigned int mb_length = tokenizer_mblength(tk, length);
   if(tk->f_callback)
     tk->f_callback(tk, type, length, tk->callback_data);
   tk->scan.cursor += length;
+  tk->scan.mb_cursor += mb_length;
 }
 
 static VALUE tokenizer_initialize_method(VALUE self)
@@ -657,6 +670,8 @@ static VALUE tokenizer_tokenize_method(VALUE self, VALUE source)
   c_source = StringValueCStr(source);
   tk->scan.cursor = 0;
   tk->scan.length = strlen(c_source);
+  tk->scan.enc_index = rb_enc_get_index(source);
+  tk->scan.mb_cursor = 0;
 
   old = tk->scan.string;
   REALLOC_N(tk->scan.string, char, tk->scan.length+1);

data/ext/html_tokenizer_ext/tokenizer.h

@@ -43,6 +43,9 @@ struct scan_t {
   char *string;
   long unsigned int cursor;
   long unsigned int length;
+
+  int enc_index;
+  long unsigned int mb_cursor;
 };
 
 struct tokenizer_t

data/html_tokenizer.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name    = "html_tokenizer"
-  spec.version = "0.0.1"
+  spec.version = "0.0.2"
   spec.summary = "HTML Tokenizer"
   spec.author  = "Francois Chagnon"
 

data/test/unit/parser_test.rb

@@ -431,29 +431,6 @@ class HtmlTokenizer::ParserTest < Minitest::Test
       tokens << token
     end
     assert_equal [[:text, 0, 4, 1, 0], [:text, 34, 38, 5, 0]], tokens
-    assert_equal "bar\n", @parser.extract(34, 38)
-  end
-
-  def test_extract_method
-    parse("abcdefg")
-    assert_equal "a", @parser.extract(0, 1)
-    assert_equal "cd", @parser.extract(2, 4)
-  end
-
-  def test_extract_method_raises_argument_error_end_past_length
-    parse("abcdefg")
-    e = assert_raises(ArgumentError) do
-      @parser.extract(0, 32)
-    end
-    assert_equal "'end' argument not in range of document", e.message
-  end
-
-  def test_extract_method_raises_argument_error_end_less_than_start
-    parse("abcdefg")
-    e = assert_raises(ArgumentError) do
-      @parser.extract(1, 0)
-    end
-    assert_equal "'end' must be greater or equal than 'start'", e.message
   end
 
   def test_solidus_or_tag_name_error
@@ -534,6 +511,29 @@ class HtmlTokenizer::ParserTest < Minitest::Test
     assert_equal 11, @parser.errors.first.column
   end
 
+  def test_attribute_with_mutlibyte_characters
+    data = ["<div title", "='your store’s'>"]
+    tokens = []
+    parse(*data) { |name, start, stop| tokens << [name, start, stop, data.join[start...stop]] }
+    assert_equal "div", @parser.tag_name
+    assert_equal "title", @parser.attribute_name
+    assert_equal "your store’s", @parser.attribute_value
+    assert_equal data.join, @parser.document
+    assert_equal data.join.size, @parser.document_length
+    assert_equal data.join.size, @parser.column_number
+    assert_equal [
+      [:tag_start, 0, 1, "<"],
+      [:tag_name, 1, 4, "div"],
+      [:whitespace, 4, 5, " "],
+      [:attribute_name, 5, 10, "title"],
+      [:equal, 10, 11, "="],
+      [:attribute_quoted_value_start, 11, 12, "'"],
+      [:attribute_quoted_value, 12, 24, "your store’s"],
+      [:attribute_quoted_value_end, 24, 25, "'"],
+      [:tag_end, 25, 26, ">"],
+    ], tokens
+  end
+
   def test_valid_syntaxes
     parse(
       '<div>',

data/test/unit/tokenizer_test.rb

@@ -324,13 +324,34 @@ class HtmlTokenizer::TokenizerTest < Minitest::Test
     ], result
   end
 
+  def test_html_with_mutlibyte_characters
+    data = "<div title='your store’s'>foo</div>"
+    result = tokenize(data)
+    assert_equal [
+      [:tag_start, "<"],
+      [:tag_name, "div"],
+      [:whitespace, " "],
+      [:attribute_name, "title"],
+      [:equal, "="],
+      [:attribute_quoted_value_start, "'"],
+      [:attribute_quoted_value, "your store’s"],
+      [:attribute_quoted_value_end, "'"],
+      [:tag_end, ">"],
+      [:text, "foo"],
+      [:tag_start, "<"],
+      [:solidus, "/"],
+      [:tag_name, "div"],
+      [:tag_end, ">"],
+    ], result
+  end
+
   private
 
   def tokenize(*parts)
     tokens = []
     @tokenizer = HtmlTokenizer::Tokenizer.new
     parts.each do |part|
-      @tokenizer.tokenize(part) { |name, start, stop| tokens << [name, part[start..(stop-1)]] }
+      @tokenizer.tokenize(part) { |name, start, stop| tokens << [name, part[start...stop]] }
     end
     tokens
   end

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: html_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Francois Chagnon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-26 00:00:00.000000000 Z
+date: 2017-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -60,8 +60,8 @@ extensions:
 - ext/html_tokenizer_ext/extconf.rb
 extra_rdoc_files: []
 files:
-- .autotest
-- .gitignore
+- ".autotest"
+- ".gitignore"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -90,17 +90,17 @@ require_paths:
 - ext
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: HTML Tokenizer