html_tokenizer 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 18ea0aa757ca8640a78cb619149b306aba21b8cc
-  data.tar.gz: 182b3fab9d73373ae04c45beb3d73eb0798bfb3b
+  metadata.gz: a45a481e7310c22092c48de49f62315d7ae19700
+  data.tar.gz: fda14319ce3a8b6770c074b68384e3c7e16d3fa7
 SHA512:
-  metadata.gz: d92c656e1c755d7b6e338732444327481af753145ca981e5e35510c44f77eecc5b06cb9eae16b02874f502037ecbf286f817886cb4fe1a5ea196fe77c760feaf
-  data.tar.gz: a57323a2228866046de1176f28f70c0309f24cb8a5fec6e0b486a4c64beaab83dcc32ab059b044c85776edcc1833bf59bb068606fcbd32c7c9ead6ecd032edaa
+  metadata.gz: b55f2ae076aa4cbf3b55fca42435bcfa63ec4e9e032a91ad55260cf0b6cc74ae75b484b0c7ad70cb08a6bf3fe75d9499feec9df768f85fdac2fdfe29cc8262ca
+  data.tar.gz: 8e3726c0471524c66270d8ed187a807b5c6bf906312e754c69e0e4da915db46ee28729f0400347413ddaf147c48a21fd4d628454e01f09eec2a104ace6452d17

data/.gitignore

@@ -30,6 +30,7 @@ tmp/
 *.i*86
 *.x86_64
 *.hex
+*.gem
 
 # Debug files
 *.dSYM/

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    html_tokenizer (0.0.3)
+    html_tokenizer (0.0.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
     minitest (5.9.0)
-    rake (11.1.2)
+    rake (12.3.0)
     rake-compiler (0.9.9)
       rake
 

data/ext/html_tokenizer_ext/extconf.rb

@@ -2,5 +2,11 @@ require 'mkmf'
 
 $CXXFLAGS += " -std=c++11 "
 $CXXFLAGS += " -g -Og -ggdb "
+$CFLAGS += " -g -Og -ggdb "
+
+if ENV['DEBUG']
+  $CXXFLAGS += "  -DDEBUG "
+  $CFLAGS += "  -DDEBUG "
+end
 
 create_makefile('html_tokenizer_ext')

data/ext/html_tokenizer_ext/parser.c

@@ -14,6 +14,7 @@ static void parser_free(void *ptr)
   size_t i;
 
   if(parser) {
+    tokenizer_free_members(&parser->tk);
     if(parser->doc.data) {
       DBG_PRINT("parser=%p xfree(parser->doc.data) %p", parser, parser->doc.data);
       xfree(parser->doc.data);
@@ -543,12 +544,12 @@ static VALUE parser_append_data(VALUE self, VALUE source, int is_placeholder)
   }
   else {
     parser->tk.scan.cursor = cursor;
-    parser->tk.scan.string = parser->doc.data;
-    parser->tk.scan.length = parser->doc.length;
+    tokenizer_set_scan_string(&parser->tk, parser->doc.data, parser->doc.length);
     parser->tk.scan.enc_index = parser->doc.enc_index;
     parser->tk.scan.mb_cursor = mb_cursor;
 
     tokenizer_scan_all(&parser->tk);
+    tokenizer_free_scan_string(&parser->tk);
   }
 
   return Qtrue;

data/ext/html_tokenizer_ext/tokenizer.c

@@ -12,16 +12,7 @@ static void tokenizer_free(void *ptr)
 {
   struct tokenizer_t *tk = ptr;
   if(tk) {
-    if(tk->current_tag) {
-      DBG_PRINT("tk=%p xfree(tk->current_tag) %p", tk, tk->current_tag);
-      xfree(tk->current_tag);
-      tk->current_tag = NULL;
-    }
-    if(tk->scan.string) {
-      DBG_PRINT("tk=%p xfree(tk->scan.string) %p", tk, tk->scan.string);
-      xfree(tk->scan.string);
-      tk->scan.string = NULL;
-    }
+    tokenizer_free_members(tk);
     DBG_PRINT("tk=%p xfree(tk)", tk);
     xfree(tk);
   }
@@ -75,6 +66,21 @@ void tokenizer_init(struct tokenizer_t *tk)
   return;
 }
 
+void tokenizer_free_members(struct tokenizer_t *tk)
+{
+  if(tk->current_tag) {
+    DBG_PRINT("tk=%p xfree(tk->current_tag) %p", tk, tk->current_tag);
+    xfree(tk->current_tag);
+    tk->current_tag = NULL;
+  }
+  if(tk->scan.string) {
+    DBG_PRINT("tk=%p xfree(tk->scan.string) %p", tk, tk->scan.string);
+    xfree(tk->scan.string);
+    tk->scan.string = NULL;
+  }
+  return;
+}
+
 VALUE token_type_to_symbol(enum token_type type)
 {
   switch(type) {
@@ -656,11 +662,30 @@ void tokenizer_scan_all(struct tokenizer_t *tk)
   return;
 }
 
+void tokenizer_set_scan_string(struct tokenizer_t *tk, const char *string, long unsigned int length)
+{
+  const char *old = tk->scan.string;
+  REALLOC_N(tk->scan.string, char, string ? length + 1 : 0);
+  DBG_PRINT("tk=%p realloc(tk->scan.string) %p -> %p length=%lu", tk, old,
+    tk->scan.string, length + 1);
+  if(string && length > 0) {
+    strncpy(tk->scan.string, string, length);
+    tk->scan.string[length] = 0;
+  }
+  tk->scan.length = length;
+  return;
+}
+
+void tokenizer_free_scan_string(struct tokenizer_t *tk)
+{
+  tokenizer_set_scan_string(tk, NULL, 0);
+  return;
+}
+
 static VALUE tokenizer_tokenize_method(VALUE self, VALUE source)
 {
   struct tokenizer_t *tk = NULL;
   char *c_source;
-  char *old;
 
   if(NIL_P(source))
     return Qnil;
@@ -670,21 +695,13 @@ static VALUE tokenizer_tokenize_method(VALUE self, VALUE source)
 
   c_source = StringValueCStr(source);
   tk->scan.cursor = 0;
-  tk->scan.length = strlen(c_source);
+  tokenizer_set_scan_string(tk, c_source, strlen(c_source));
   tk->scan.enc_index = rb_enc_get_index(source);
   tk->scan.mb_cursor = 0;
 
-  old = tk->scan.string;
-  REALLOC_N(tk->scan.string, char, tk->scan.length+1);
-  DBG_PRINT("tk=%p realloc(tk->scan.string) %p -> %p length=%lu", tk, old,
-    tk->scan.string,  tk->scan.length+1);
-  strncpy(tk->scan.string, c_source, tk->scan.length);
-
   tokenizer_scan_all(tk);
 
-  DBG_PRINT("tk=%p xfree(tk->scan.string) 0x%p", tk, tk->scan.string);
-  xfree(tk->scan.string);
-  tk->scan.string = NULL;
+  tokenizer_free_scan_string(tk);
 
   return Qtrue;
 }

data/ext/html_tokenizer_ext/tokenizer.h

@@ -70,6 +70,9 @@ struct tokenizer_t
 
 void Init_html_tokenizer_tokenizer(VALUE mHtmlTokenizer);
 void tokenizer_init(struct tokenizer_t *tk);
+void tokenizer_free_members(struct tokenizer_t *tk);
+void tokenizer_set_scan_string(struct tokenizer_t *tk, const char *string, long unsigned int length);
+void tokenizer_free_scan_string(struct tokenizer_t *tk);
 void tokenizer_scan_all(struct tokenizer_t *tk);
 VALUE token_type_to_symbol(enum token_type type);
 

data/html_tokenizer.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name    = "html_tokenizer"
-  spec.version = "0.0.3"
+  spec.version = "0.0.5"
   spec.summary = "HTML Tokenizer"
   spec.author  = "Francois Chagnon"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Francois Chagnon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-21 00:00:00.000000000 Z
+date: 2017-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake