java_bin 0.3.0 → 0.3.1

data/CHANGELOG

@@ -1,3 +1,6 @@
+0.3.1 (2010/01/22)
+  * performance improve (shared and frozen string when extern string).
+
 0.3.0 (2010/01/19)
   * windows build support
 

data/README.rdoc

@@ -4,7 +4,7 @@ This is an Apache Solr JavaBin format (binary format) implementation for Ruby.
 
 == Features
 
-* fast parse, and less network traffic.
+* fast parse(2.5 - 5 times faster than ruby eval), and less network traffic.
  MRI 1.8.7
   [data1]
   ruby eval parse. 5000 times. elapsed time 1.282744
@@ -45,8 +45,8 @@ This is an Apache Solr JavaBin format (binary format) implementation for Ruby.
 == Requirements
 
 * Ruby1.8.7 or later (include 1.9.x)
-* (recommended) C compiler: you can also use java_bin without c extension, but 'pure' is 30 times slower than 'ext'.
-* JavaBin has been tested with MRI 1.8.7, REE 1.8.7, YARV 1.9.2 pre1 on Ubuntu Linux 9.10 (32bit) and MRI 1.8.6 on Windows Vista Business (32bit), and Apache Solr 1.4  
+* (recommended) C compiler (gcc or vc++): you can also use java_bin without c extension, but 'pure' is 30 times slower than 'ext'.
+* JavaBin has been tested with MRI 1.8.7, REE 1.8.7, YARV 1.9.2 pre1 on Ubuntu Linux 9.10 (32bit) and MRI 1.8.6 on Windows Vista (32bit), and Apache Solr 1.4  
 
 == Install
 
@@ -55,6 +55,11 @@ This is an Apache Solr JavaBin format (binary format) implementation for Ruby.
  gem install java_bin
 
  (Windows)
+ fix (path to)\ruby\lib\ruby\1.8\i386-mswin32\config.h
+   #if _MSC_VER != 1200
+     ↓
+   #if _MSC_VER < 1200
+
  start => visual studio c++ 200x xxxx edition => visual studio tools => visual studio 200x command prompt
  gem sources -a http://gemcutter.org
  gem install java_bin
@@ -86,6 +91,8 @@ This is an Apache Solr JavaBin format (binary format) implementation for Ruby.
 
 * more parse speed
 * license
+* pure parse encoding bug (1.9.x)
+* shared string over parsing
 * windows build(1.9.x)
 * 64bit build
 * builder(writer)作成

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/ext/java_bin/ext/parser.c

@@ -72,11 +72,11 @@ static VALUE JavaBinParser_read_small_long(JAVA_BIN_PARSER* ptr) {
 static VALUE JavaBinParser_read_string(JAVA_BIN_PARSER* ptr) {
   int size;
   int i;
-
+  int start;
   unsigned char b;
 
   size = JavaBinParser_read_size(ptr);
-  ptr->last_string_offset = ptr->current;
+  start = ptr->current;
   for (i = 0; i < size; i++) {
     /* HINT. read utf-8 char */
     b = _getbyte(ptr);
@@ -87,8 +87,7 @@ static VALUE JavaBinParser_read_string(JAVA_BIN_PARSER* ptr) {
       _skipbytes(ptr, 2);
     } /* TODO 4byte以上のケース? */
   }
-  ptr->last_string_len = ptr->current - ptr->last_string_offset;
-  return _utf8_string((const char*) &ptr->data[ptr->last_string_offset], ptr->last_string_len); 
+  return _utf8_string((const char*) &ptr->data[start], ptr->current - start); 
 }
 
 static VALUE JavaBinParser_read_byte(JAVA_BIN_PARSER* ptr) {
@@ -140,7 +139,7 @@ static VALUE JavaBinParser_read_double(JAVA_BIN_PARSER* ptr) {
 }
 
 static void JavaBinParser_extend_cache(JAVA_BIN_PARSER* ptr) {
-  _EXTERN_STRING_INFO* newP;
+  VALUE* newP;
   int next_size;
   if (ptr->cache == NULL) {
     next_size = 64;
@@ -148,13 +147,13 @@ static void JavaBinParser_extend_cache(JAVA_BIN_PARSER* ptr) {
     next_size = ptr->cache_size * 2;
   }
 
-  newP = (_EXTERN_STRING_INFO*) malloc(next_size * sizeof(_EXTERN_STRING_INFO));
+  newP = (VALUE*) malloc(next_size * sizeof(VALUE));
   if (!newP) {
     rb_raise(rb_eRuntimeError, "JavaBinParser_extend_cache - allocate error");
   }
 
   if (ptr->cache) {
-    memcpy(newP, ptr->cache, sizeof(_EXTERN_STRING_INFO) * ptr->cache_size);
+    memcpy(newP, ptr->cache, sizeof(VALUE) * ptr->cache_size);
   }
   ptr->cache = newP;
   ptr->cache_size = next_size;
@@ -186,17 +185,19 @@ static VALUE JavaBinParser_read_val(JAVA_BIN_PARSER* ptr) {
         /* rubyの文字列 */
         value = JavaBinParser_read_val(ptr);
 
-        /* 参照文字列としてcの文字列を保持 */
-        ptr->cache[ptr->cache_index].offset = ptr->last_string_offset;
-        ptr->cache[ptr->cache_index].len    = ptr->last_string_len;
-        ptr->cache_index ++;
+        /* 参照文字列として文字列を保持 */
+        ptr->cache[ptr->cache_index++] = value;
         /* 参照文字列用のcacheを拡張する */
         if (ptr->cache_size <= ptr->cache_index) {
           JavaBinParser_extend_cache(ptr);
         }
         return value;
       } else {
-        return _utf8_string((const char*)&ptr->data[ptr->cache[size - 1].offset], ptr->cache[size - 1].len); 
+        return rb_str_new4(ptr->cache[size - 1]);   // freeze共有
+        //return rb_str_new3(ptr->cache[size - 1]); // 共有(変更があったら分裂)
+        
+	//return ptr->cache[size - 1];              // 同じ物
+        //return rb_str_dup(ptr->cache[size - 1]);  // コピー
       }
     case SHIFTED_ORDERED_MAP:
     case SHIFTED_NAMED_LST:
@@ -268,6 +269,7 @@ static VALUE JavaBinParser_read_val(JAVA_BIN_PARSER* ptr) {
     case SOLRDOCLST:
       hash = rb_hash_new();
       value = JavaBinParser_read_val(ptr);
+      // TODO キーのfreeze
       rb_hash_aset(hash, rb_str_new2("numFound"), rb_ary_entry(value, 0));
       rb_hash_aset(hash, rb_str_new2("start"),    rb_ary_entry(value, 1));
       rb_hash_aset(hash, rb_str_new2("maxScore"), rb_ary_entry(value, 2));
@@ -288,8 +290,17 @@ static void JavaBinParser_free(JAVA_BIN_PARSER* ptr) {
   }
 }
 
+static void JavaBinParser_mark(JAVA_BIN_PARSER* ptr) {
+  int i;
+  if (ptr) {
+    for (i = 0; i < ptr->cache_index; i++) {
+      rb_gc_mark_maybe(ptr->cache[i]);
+    }
+  }
+}
+
 static VALUE JavaBinParser_alloc(VALUE klass) {
-  return Data_Wrap_Struct(klass, 0, JavaBinParser_free, NULL);
+  return Data_Wrap_Struct(klass, JavaBinParser_mark, JavaBinParser_free, NULL);
 }
 
 /*
@@ -358,7 +369,7 @@ static VALUE rb_cParser_initialize(VALUE self) {
   }
   DATA_PTR(self) = ptr;
 
-  /* 参照文字列の準備(ここでも初期化しておかないと、たまにsegしちゃいますruby 1.8.7) */
+  /* 参照文字列の準備(ここでも初期化しておかないと、たまにsegvしちゃいますruby 1.8.7) */
   ptr->cache = NULL;
   ptr->cache_index = 0;
  

data/ext/java_bin/ext/parser.h

@@ -55,14 +55,6 @@
 #define  SHIFTED_SINT            (SINT >> 5)
 #define  SHIFTED_SLONG           (SLONG >> 5)
 
-/*
- * 参照文字列情報保持用
- */
-typedef struct _extern_string_info {
-  int offset;
-  int len;
-} _EXTERN_STRING_INFO;
-
 /*
  * 読込処理データ保持構造体
  */
@@ -73,11 +65,9 @@ typedef struct java_bin_parser {
   unsigned char  tag_byte;
 
   /* 外部文字列用 */
-  _EXTERN_STRING_INFO* cache; 
+  VALUE*               cache; 
   int                  cache_size;
   int                  cache_index;
-  int                  last_string_offset;
-  int                  last_string_len;
 } JAVA_BIN_PARSER;
 
 #ifdef HAVE_RUBY_ENCODING_H
@@ -137,6 +127,7 @@ static VALUE JavaBinParser_read_float(JAVA_BIN_PARSER* ptr);
 static VALUE JavaBinParser_read_double(JAVA_BIN_PARSER* ptr);
 static VALUE JavaBinParser_read_val(JAVA_BIN_PARSER* ptr);
 static void JavaBinParser_free(JAVA_BIN_PARSER* ptr);
+static void JavaBinParser_mark(JAVA_BIN_PARSER* ptr);
 static VALUE JavaBinParser_alloc(VALUE klass);
 static void JavaBinParser_extend_cache(JAVA_BIN_PARSER* ptr);
 static VALUE rb_cParser_parse(VALUE self, VALUE data);

data/java_bin.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{java_bin}
-  s.version = "0.3.0"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["kennyj"]
-  s.date = %q{2010-01-20}
+  s.date = %q{2010-01-22}
   s.description = %q{Apache Solr JavaBin format (binary format) implementation for Ruby.}
   s.email = %q{kennyj@gmail.com}
   s.extensions = ["ext/java_bin/ext/extconf.rb"]

data/test/test_java_bin_parser.rb

@@ -238,7 +238,9 @@ class TestJavaBinParser < Test::Unit::TestCase
 
   def test_ordered_map
     arr = [1, (5 << 5) | 2] + [(1 << 5) | 1] + "a".unpack("C*") + [3, 8] + [(1 << 5) | 1] + "b".unpack("C*") + [3, 9]
-    assert_equal({"a" => 8, "b" => 9}, @parser.parse(arr.pack("C*")))
+    result = @parser.parse(arr.pack("C*"))
+    assert_equal({"a" => 8, "b" => 9}, result)
+    result.each { |k,v| assert k.frozen? }
   end
 
 #  def test_named_lst
@@ -250,7 +252,11 @@ class TestJavaBinParser < Test::Unit::TestCase
           [(7 << 5) | 0] + [(1 << 5) | 3] + "あいa".unpack("C*") +
           [(1 << 5) | 1] + "b".unpack("C*") +
           [(7 << 5) | 1]
-    assert_equal({"a" => "あいa", "b" => "あいa"}, @parser.parse(arr.pack("C*")))
+    result = @parser.parse(arr.pack("C*"))
+    assert_equal({"a" => "あいa", "b" => "あいa"}, result)
+
+    assert_equal 'UTF-8', result['a'].encoding.to_s if result['a'].respond_to? :encoding
+    assert_equal 'UTF-8', result['b'].encoding.to_s if result['b'].respond_to? :encoding
   end
 
   LARGE_SIZE = 1000

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: java_bin
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors: 
 - kennyj
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-20 00:00:00 +09:00
+date: 2010-01-22 00:00:00 +09:00
 default_executable: 
 dependencies: []