java_bin 0.1.0

data/fixtures/json.dat

@@ -0,0 +1,55 @@
+{
+ "responseHeader":{
+  "status":0,
+  "QTime":0,
+  "params":{
+	"explainOther":"",
+	"fl":"*,score",
+	"indent":"on",
+	"start":"0",
+	"q":"software",
+	"hl.fl":"",
+	"qt":"standard",
+	"wt":"json",
+	"fq":"",
+	"version":"2.2",
+	"rows":"10"}},
+ "response":{"numFound":2,"start":0,"maxScore":0.6288448,"docs":[
+	{
+	 "id":"UTF8TEST",
+	 "name":"Test with some UTF-8 encoded characters",
+	 "manu":"Apache Software Foundation",
+	 "price":0.0,
+	 "inStock":true,
+	 "cat":[
+	  "software",
+	  "search"],
+	 "features":[
+	  "No accents here",
+	  "This is an e acute: é",
+	  "eaiou with circumflexes: êâîôû",
+	  "eaiou with umlauts: ëäïöü",
+	  "tag with escaped chars: <nicetag/>",
+	  "escaped ampersand: Bonnie & Clyde"],
+	 "score":0.6288448},
+	{
+	 "id":"SOLR1000",
+	 "name":"Solr, the Enterprise Search Server",
+	 "manu":"Apache Software Foundation",
+	 "price":0.0,
+	 "popularity":10,
+	 "inStock":true,
+	 "incubationdate_dt":"2006-01-17T00:00:00Z",
+	 "cat":[
+	  "software",
+	  "search"],
+	 "features":[
+	  "Advanced Full-Text Search Capabilities using Lucene",
+	  "Optimized for High Volume Web Traffic",
+	  "Standards Based Open Interfaces - XML and HTTP",
+	  "Comprehensive HTML Administration Interfaces",
+	  "Scalability - Efficient Replication to other Solr Search Servers",
+	  "Flexible and Adaptable with XML configuration and Schema",
+	  "Good unicode support: héllo (hello with an accent over the e)"],
+	 "score":0.50307584}]
+ }}

data/fixtures/ruby.dat

@@ -0,0 +1,55 @@
+{
+ 'responseHeader'=>{
+  'status'=>0,
+  'QTime'=>0,
+  'params'=>{
+	'explainOther'=>'',
+	'fl'=>'*,score',
+	'indent'=>'on',
+	'start'=>'0',
+	'q'=>'software',
+	'hl.fl'=>'',
+	'qt'=>'standard',
+	'wt'=>'ruby',
+	'fq'=>'',
+	'version'=>'2.2',
+	'rows'=>'10'}},
+ 'response'=>{'numFound'=>2,'start'=>0,'maxScore'=>0.6288448,'docs'=>[
+	{
+	 'id'=>'UTF8TEST',
+	 'name'=>'Test with some UTF-8 encoded characters',
+	 'manu'=>'Apache Software Foundation',
+	 'price'=>0.0,
+	 'inStock'=>true,
+	 'cat'=>[
+	  'software',
+	  'search'],
+	 'features'=>[
+	  'No accents here',
+	  'This is an e acute: é',
+	  'eaiou with circumflexes: êâîôû',
+	  'eaiou with umlauts: ëäïöü',
+	  'tag with escaped chars: <nicetag/>',
+	  'escaped ampersand: Bonnie & Clyde'],
+	 'score'=>0.6288448},
+	{
+	 'id'=>'SOLR1000',
+	 'name'=>'Solr, the Enterprise Search Server',
+	 'manu'=>'Apache Software Foundation',
+	 'price'=>0.0,
+	 'popularity'=>10,
+	 'inStock'=>true,
+	 'incubationdate_dt'=>'2006-01-17T00:00:00Z',
+	 'cat'=>[
+	  'software',
+	  'search'],
+	 'features'=>[
+	  'Advanced Full-Text Search Capabilities using Lucene',
+	  'Optimized for High Volume Web Traffic',
+	  'Standards Based Open Interfaces - XML and HTTP',
+	  'Comprehensive HTML Administration Interfaces',
+	  'Scalability - Efficient Replication to other Solr Search Servers',
+	  'Flexible and Adaptable with XML configuration and Schema',
+	  'Good unicode support: héllo (hello with an accent over the e)'],
+	 'score'=>0.50307584}]
+ }}

data/java_bin.gemspec

@@ -0,0 +1,67 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{java_bin}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["kennyj"]
+  s.date = %q{2010-01-11}
+  s.description = %q{Solr JavaBin format implementation for Ruby.}
+  s.email = %q{kennyj@gmail.com}
+  s.extensions = ["ext/java_bin/ext/extconf.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc",
+     "TODO"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "TODO",
+     "VERSION",
+     "ext/java_bin/ext/extconf.rb",
+     "ext/java_bin/ext/parser.c",
+     "ext/java_bin/ext/parser.h",
+     "fixtures/javabin.dat",
+     "fixtures/json.dat",
+     "fixtures/ruby.dat",
+     "java_bin.gemspec",
+     "lib/java_bin.rb",
+     "lib/java_bin/ext.rb",
+     "lib/java_bin/ext/.keep",
+     "lib/java_bin/pure.rb",
+     "lib/java_bin/pure/parser.rb",
+     "lib/java_bin/version.rb",
+     "test/helper.rb",
+     "test/test_java_bin_parser.rb",
+     "test/xxx_performance.rb"
+  ]
+  s.homepage = %q{http://github.com/kennyj/java_bin}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib", "ext"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Solr JavaBin format implementation for Ruby.}
+  s.test_files = [
+    "test/xxx_performance.rb",
+     "test/test_java_bin_parser.rb",
+     "test/helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/lib/java_bin/ext.rb

@@ -0,0 +1,9 @@
+# vim:fileencoding=utf-8
+module JavaBin
+  module Ext
+    require 'java_bin/ext/parser'
+    #require 'parser'
+    $DEBUG and warn "Using c extension for JavaBin."
+    ::JavaBin.parser = ::JavaBin::Ext::Parser
+  end
+end

data/lib/java_bin/pure/parser.rb

@@ -0,0 +1,219 @@
+# vim:fileencoding=utf-8
+module JavaBin
+  module Pure
+  NULL       = 0
+  BOOL_TRUE  = 1
+  BOOL_FALSE = 2
+  BYTE       = 3
+  SHORT      = 4
+  DOUBLE     = 5
+  INT        = 6
+  LONG       = 7
+  FLOAT      = 8
+  DATE       = 9
+  MAP        = 10
+  SOLRDOC    = 11
+  SOLRDOCLST = 12
+  BYTEARR    = 13
+  ITERATOR   = 14
+  TERM       = 15 #END = 15
+
+  #TAG_AND_LEN  = (1 << 5)
+  STR           = (1 << 5)
+  SINT          = (2 << 5)
+  SLONG         = (3 << 5)
+  ARR           = (4 << 5)
+  ORDERED_MAP   = (5 << 5)
+  NAMED_LST     = (6 << 5)
+  EXTERN_STRING = (7 << 5)
+
+  # TODO 論理シフト
+  SHIFTED_STR           = STR >> 5
+  SHIFTED_ARR           = ARR >> 5
+  SHIFTED_EXTERN_STRING = EXTERN_STRING >> 5
+  SHIFTED_ORDERED_MAP   = ORDERED_MAP >> 5
+  SHIFTED_NAMED_LST     = NAMED_LST >> 5
+  SHIFTED_SINT          = SINT >> 5
+  SHIFTED_SLONG         = SLONG >> 5
+
+  VERSION  = 1
+  TERM_OBJ = :term_obj
+ 
+    class Parser
+  
+      attr_reader :tag_byte, :input, :current
+    
+      def initialize
+      end
+    
+      def parse(input)
+        array = input.bytes.to_a
+        check_version(array[0])
+        @input = array
+        @current = 1 # HINT VERSIONをとばす
+        @tag_byte = nil
+        read_val
+      end
+    
+      private
+      def check_version(byte)
+        return true if VERSION == byte
+        raise "unsupported version #{byte}"
+      end
+   
+      def getbyte
+        ret = @input[@current]
+        @current += 1
+        ret
+      end
+      
+      def getbytes(size)
+        ret = @input[@current...(@current + size)]
+        @current += size
+        ret
+      end
+    
+      def read_val
+        @tag_byte = getbyte
+        case (@tag_byte >> 5) # TODO 論理シフト
+          when SHIFTED_STR
+            return read_chars
+          when SHIFTED_ARR
+            size = read_size
+            array = Array.new(size)
+            size.times { |i| array[i] = read_val }
+            return array
+          when SHIFTED_EXTERN_STRING
+            size = read_size
+            if size == 0
+              str = read_val
+              @exts ||= []
+              @exts << str
+              return str
+            else
+              return @exts[size - 1].dup
+            end
+          when SHIFTED_ORDERED_MAP, SHIFTED_NAMED_LST
+            size = read_size
+            hash = {}
+            size.times do
+              k = read_val
+              v = read_val
+              hash[k] = v
+            end
+            return hash
+          when SHIFTED_SINT
+            return read_small_int
+          when SHIFTED_SLONG
+            return read_small_int
+        end
+    
+        case @tag_byte
+          when NULL
+            return nil
+          when BOOL_TRUE
+            return true
+          when BOOL_FALSE
+            return false
+          when BYTE
+            return getbytes(1).pack("C*").unpack("c")[0]
+          when SHORT
+            return getbytes(2).reverse.pack("C*").unpack("s")[0]
+          when DOUBLE
+            return getbytes(8).pack("C*").unpack("G")[0]
+          when INT
+            return getbytes(4).reverse.pack("C*").unpack("i")[0]
+          when LONG
+            return getbytes(8).reverse.pack("C*").unpack("q")[0]
+          when FLOAT
+            return getbytes(4).pack("C*").unpack("g")[0]
+          when DATE
+            x = getbytes(8).reverse.pack("C*").unpack("q")[0]
+            return Time.at(x/1000)
+          when MAP
+            size = read_v_int
+            hash = {}
+            size.times do
+              k = read_val
+              v = read_val
+              hash[k] = v
+            end
+            return hash
+          when BYTEARR
+            size = read_v_int
+            return getbytes(size)
+          when ITERATOR
+            array = []
+            while true
+              i = read_val
+              break if i == TERM_OBJ
+              array << i
+            end
+            return array
+          when TERM
+            return TERM_OBJ
+          when SOLRDOC
+            return read_val
+          when SOLRDOCLST
+            result = {}
+            list = read_val
+            result['numFound'] = list[0]
+            result['start']    = list[1]
+            result['maxScore'] = list[2]
+            result['docs'] = read_val
+            return result
+        end
+      end
+    
+      def read_v_int
+        byte = getbyte
+        result = byte & 0x7f
+        shift = 7
+        while (byte & 0x80) != 0
+          byte = getbyte
+          result |= ((byte & 0x7f) << shift)
+          shift += 7
+        end
+        result 
+      end
+    
+      def read_size
+        size = (@tag_byte & 0x1f)
+        size += read_v_int if (size == 0x1f)
+        size
+      end
+    
+      def read_small_int
+        result = @tag_byte & 0x0F
+        result = ((read_v_int << 4) | result) if ((@tag_byte & 0x10) != 0)
+        result
+      end
+    
+      def read_chars
+        size = read_size
+        str = ''
+        size.times {
+          # HINT. read utf-8 char
+          b = getbyte
+          if ((b & 0x80) == 0)
+            str << b
+          elsif ((b & 0xE0) == 0xC0)
+            #str << (((b & 0x1F) << 6) | (getbyte & 0x3F))
+            str << b
+            str << getbyte
+          else
+            #str << (((b & 0x0F) << 12) | ((getbyte & 0x3F) << 6) | (getbyte & 0x3F))
+            str << b
+            str << getbyte
+            str << getbyte
+          end
+        }
+        str.force_encoding('utf-8') if str.respond_to? :force_encoding
+        str
+      end
+  
+    end
+  
+  end
+end
+

data/lib/java_bin/pure.rb

@@ -0,0 +1,8 @@
+# vim:fileencoding=utf-8
+module JavaBin
+  module Pure
+    require 'java_bin/pure/parser'
+    $DEBUG and warn "Using pure for JavaBin."
+    ::JavaBin.parser = ::JavaBin::Pure::Parser
+  end
+end

data/lib/java_bin/version.rb

@@ -0,0 +1,10 @@
+# vim:fileencoding=utf-8
+module JavaBin
+  # JavaBin version
+  VERSION = '0.1.0'
+  VERSION_ARRAY = VERSION.split(/\./).map { |x| x.to_i } # :nodoc:
+  VERSION_MAJOR = VERSION_ARRAY[0] # :nodoc:
+  VERSION_MINOR = VERSION_ARRAY[1] # :nodoc:
+  VERSION_BUILD = VERSION_ARRAY[2] # :nodoc: 
+end
+

data/lib/java_bin.rb

@@ -0,0 +1,17 @@
+# vim:fileencoding=utf-8
+module JavaBin
+  def self.parser=(value)
+    @parser = value
+  end
+  def self.parser
+    @parser
+  end
+ 
+  require 'java_bin/version'
+  begin
+    require 'java_bin/ext'
+  rescue LoadError => e
+    require 'java_bin/pure'
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+# vim:fileencoding=utf-8
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'java_bin'
+
+class Test::Unit::TestCase
+end

data/test/test_java_bin_parser.rb

@@ -0,0 +1,240 @@
+# vim:fileencoding=utf-8
+require 'helper'
+
+case ENV['JavaBin']
+when 'pure'
+then
+  require 'java_bin/pure'
+else
+  require 'java_bin/ext'
+end
+
+class TestJavaBin < Test::Unit::TestCase
+
+  private
+  def write_v_int(i, output)
+    while ((i & ~0x7F) != 0) 
+      output.putc(((i & 0x7f) | 0x80))
+      # i >>>= 7
+      i = (i >> 7) # TODO 論理シフト
+    end 
+    output.putc(i)
+  end
+
+  def write_tag(tag, size, output)
+    if ((tag & 0xe0) != 0)
+      if (size < 0x1f)
+        output.putc(tag | size)
+      else
+ 	output.putc(tag | 0x1f)
+ 	write_v_int(size - 0x1f, output)
+      end 
+    else
+      output.putc(tag)
+      write_v_int(size, output)
+    end
+  end
+
+  def elapsed_time(name, t, &block)
+    GC.start
+    s = Time.now
+    t.times {
+      block.call
+    }
+    e = Time.now
+    puts "#{name}#{t} times. elapsed time #{e - s}"
+    (e - s)
+  end
+
+  public
+  def setup
+    @parser = JavaBin.parser.new
+  end
+
+  def test_valid_version
+    assert @parser.parse([1, 1].pack("C*"))
+  end
+  def test_invalid_version
+    assert_raise(RuntimeError) { @parser.parse([2].pack("C*")) }
+  end
+
+  def test_javabin_dat
+    result = @parser.parse(open("fixtures/javabin.dat", "r:utf-8").read)
+    assert result['response']['docs'][0]['features'].include?('eaiou with umlauts: ëäïöü')
+    assert_equal result['response']['docs'][1]['incubationdate_dt'], Time.local(2006, 1, 17, 9, 0, 0)
+    assert_equal result['response']['docs'][1]['score'], 0.5030758380889893 
+  end
+
+  TIMES = 5000
+  def test_javabin_parse_and_ruby_eval
+    jb = open("fixtures/javabin.dat", "r:utf-8").read
+    r  = open("fixtures/ruby.dat", "r:utf-8").read
+    puts ""
+    r_et  = elapsed_time("ruby eval parse. ", TIMES) { eval(r) }
+    jb_et = elapsed_time("javabin parse.   ", TIMES) { @parser.parse(jb) }
+    assert (jb_et * 3) < r_et if @parser.is_a? JavaBin::Ext::Parser
+  end
+
+  def test_null
+    assert_nil @parser.parse([1, 0].pack("C*"))
+  end
+
+  def test_true
+    assert @parser.parse([1, 1].pack("C*"))
+  end
+
+  def test_false
+    assert !@parser.parse([1, 2].pack("C*"))
+  end
+
+  def test_byte
+    assert_equal  1,  @parser.parse([1, 3, 0x01].pack("C*"))
+    assert_equal 127, @parser.parse([1, 3, 0x7f].pack("C*"))
+    assert_equal -1,  @parser.parse([1, 3, 0xff].pack("C*"))
+    assert_equal -2,  @parser.parse([1, 3, 0xfe].pack("C*"))
+  end
+
+  def test_short
+    assert_equal  1,    @parser.parse([1, 4, 0x00, 0x01].pack("C*"))
+    assert_equal 32767, @parser.parse([1, 4, 0x7f, 0xff].pack("C*"))
+    assert_equal -1,    @parser.parse([1, 4, 0xff, 0xff].pack("C*"))
+    assert_equal -2,    @parser.parse([1, 4, 0xff, 0xfe].pack("C*"))
+  end
+
+  def test_double
+    assert_equal -1.0, @parser.parse([1, 5, -1.0].pack("C2G"))
+  end
+
+  def test_int
+    assert_equal  1,         @parser.parse([1, 6, 0x00, 0x00, 0x00, 0x01].pack("C*"))
+    assert_equal 2147483647, @parser.parse([1, 6, 0x7f, 0xff, 0xff, 0xff].pack("C*"))
+    assert_equal -1,         @parser.parse([1, 6, 0xff, 0xff, 0xff, 0xff].pack("C*"))
+    assert_equal -2,         @parser.parse([1, 6, 0xff, 0xff, 0xff, 0xfe].pack("C*"))
+  end
+
+  def test_long
+    assert_equal  1,                   @parser.parse([1, 7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01].pack("C*"))
+    assert_equal 9223372036854775807 , @parser.parse([1, 7, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff].pack("C*"))
+    assert_equal -1,                   @parser.parse([1, 7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff].pack("C*"))
+    assert_equal -2,                   @parser.parse([1, 7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe].pack("C*"))
+  end
+
+  def test_float
+    assert_equal -1.0, @parser.parse([1, 8, -1.0].pack("C2g"))
+  end
+
+  def test_date
+    t = Time.now
+    x = (t.to_f * 1000).to_i
+    x = [x].pack("q").unpack("C*").reverse # TODO endian次第なので修正必要
+    assert_equal t.to_i, @parser.parse(([1, 9] + x).pack("C*")).to_i
+  end
+
+  def test_map
+    assert_equal({0 => 0, 1 => 1}, @parser.parse([1, 10, 2, 3, 0, 3, 0, 3, 1, 3, 1].pack("C*")))
+  end
+
+  def test_solrdoc
+    arr = [1, 11] + [(5 << 5) | 2] + [(1 << 5) | 1] + "a".unpack("C*") + [3, 8] + [(1 << 5) | 1] + "b".unpack("C*") + [3, 9]
+    assert_equal({"a" => 8, "b" => 9}, @parser.parse(arr.pack("C*")))
+  end
+
+  def test_solrdoclst
+    arr = [1, 12] + [(4 << 5) | 3, 3, 3, 3, 4, 3, 5] + [(4 << 5) | 0]
+    assert_equal({'numFound' => 3, 'start' => 4, 'maxScore' => 5, 'docs' => []}, @parser.parse(arr.pack("C*")))
+  end
+
+  def test_bytearr
+    array = [0, 1, 0xff]
+    assert_equal array, @parser.parse([1, 13, array.size, *array].pack("C*"))
+  end
+
+  def test_large_bytearr
+    array = [0,1,255] * 100
+    sio = StringIO.new
+    sio.putc 1 #VERSION
+    sio.putc 13 #JavaBin::BYTEARR
+    write_v_int(array.size, sio)
+    array.each { |e| sio.putc e }
+    sio.pos = 0
+    assert_equal array, @parser.parse(sio.read) 
+  end
+
+  def test_iterator
+    assert_equal([0, 127, nil, true], @parser.parse([1, 14, 3, 0, 3, 127, 0, 1,15].pack("C*")))
+  end
+
+#  def test_term
+#  end
+
+  def test_string
+    assert_equal "あい", @parser.parse(([1, (1 << 5) | 2] + "あい".unpack("C*")).pack("C*"))
+  end
+
+  def test_long_string
+    str = "0123456789" * 100
+    sio = StringIO.new
+    sio.putc 1 #VERSION
+    write_tag(1 << 5, str.size, sio)
+    str.each_byte { |e| sio.putc e }
+    sio.pos = 0
+    assert_equal str, @parser.parse(sio.read)
+  end
+
+  def test_sint
+    assert_equal 8, @parser.parse([1, (2 << 5) | 8].pack("C*"))
+    # flunk("not implemented yet.")
+  end
+
+  def test_slong
+    assert_equal 8, @parser.parse([1, (3 << 5) | 8].pack("C*"))
+    # flunk("not implemented yet.")
+  end
+
+  def test_arr
+    assert_equal [3, 4], @parser.parse([1, (4 << 5) | 2, 3, 3, 3, 4].pack("C*"))
+  end
+
+  def test_large_arr
+    array = [0, 1, 2, 3, 4] * 100
+    sio = StringIO.new
+    sio.putc 1 #VERSION
+    write_tag(4 << 5, array.size, sio)
+    array.each { |e| sio.putc 3; sio.putc e }
+    sio.pos = 0
+    assert_equal array, @parser.parse(sio.read)
+  end
+
+  def test_ordered_map
+    arr = [1, (5 << 5) | 2] + [(1 << 5) | 1] + "a".unpack("C*") + [3, 8] + [(1 << 5) | 1] + "b".unpack("C*") + [3, 9]
+    assert_equal({"a" => 8, "b" => 9}, @parser.parse(arr.pack("C*")))
+  end
+
+#  def test_named_lst
+#  end
+
+  def test_extern_string
+    arr = [1, (5 << 5) | 2] +
+          [(1 << 5) | 1] + "a".unpack("C*") +
+          [(7 << 5) | 0] + [(1 << 5) | 3] + "あいa".unpack("C*") +
+          [(1 << 5) | 1] + "b".unpack("C*") +
+          [(7 << 5) | 1]
+    assert_equal({"a" => "あいa", "b" => "あいa"}, @parser.parse(arr.pack("C*")))
+  end
+
+  LARGE_SIZE = 1000
+  def test_long_large_amount_extern_string
+    sio = StringIO.new
+    sio.putc 1 #VERSION
+    write_tag(4 << 5, LARGE_SIZE, sio)
+    LARGE_SIZE.times { |i|
+      sio.putc((7 << 5) | 0)
+      sio.putc((1 << 5) | 1)
+      sio.putc("a")
+      write_tag((7 << 5), i, sio)
+    }
+    sio.pos = 0
+    assert_equal ['a'] * LARGE_SIZE, @parser.parse(sio.read)
+  end
+
+end