msgpack-jruby 1.0.0-java → 1.1.3-java

data/.gitignore

@@ -1,5 +1,4 @@
 *.gem
 .bundle
-Gemfile.lock
 pkg/*
-/benchmark
+/benchmark

data/.rvmrc

@@ -1 +1 @@
-rvm --create use jruby-1.6.2@msgpack-jruby
+rvm --create use jruby-1.6.7@msgpack-jruby

data/Gemfile

@@ -1,11 +1,13 @@
 source :rubygems
 
-gemspec
+gem 'jruby-openssl'
 
 group :test do
   gem 'rspec'
   gem 'viiite'
   gem 'ffi-ncurses'
   gem 'bson'
+  gem 'bson_ext', :platforms => :mri
   gem 'json'
+  gem 'msgpack', :platforms => :mri
 end

data/Gemfile.lock

@@ -0,0 +1,52 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    alf (0.10.1)
+      highline (~> 1.6.2)
+      myrrha (~> 1.2.1)
+      quickl (~> 0.4.1)
+    bouncy-castle-java (1.5.0146.1)
+    bson (1.5.2)
+    bson (1.5.2-java)
+    bson_ext (1.5.2)
+      bson (= 1.5.2)
+    diff-lcs (1.1.3)
+    ffi (1.0.9)
+    ffi (1.0.9-java)
+    ffi-ncurses (0.3.3)
+      ffi (>= 0.6.3)
+    gnuplot (2.3.6)
+    highline (1.6.2)
+    jruby-openssl (0.7.4)
+      bouncy-castle-java
+    json (1.5.0)
+    json (1.5.0-java)
+    msgpack (0.4.6)
+    myrrha (1.2.1)
+    quickl (0.4.1)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    viiite (0.1.0)
+      alf (~> 0.10.0)
+      gnuplot (~> 2.3.6)
+      quickl (~> 0.4.0)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bson
+  bson_ext
+  ffi-ncurses
+  jruby-openssl
+  json
+  msgpack
+  rspec
+  viiite

data/Rakefile

@@ -1,22 +1,56 @@
-require 'bundler'
+$: << 'lib'
 
+require 'bundler/setup'
+require 'msgpack/version'
 
-Bundler::GemHelper.install_tasks
 
 task :clean do
-  rm Dir['lib/ext/**/*.class']
+  rm Dir['ext/java/**/*.class']
 end
 
 task :compile do
-  exec %(javac -source 1.6 -target 1.6 -cp lib/ext/msgpack-0.5.2-devel.jar:$MY_RUBY_HOME/lib/jruby.jar -d lib/ext ext/java/org/msgpack/**/*.java)
+  classpath = (Dir["lib/ext/*.jar"] + ["#{ENV['MY_RUBY_HOME']}/lib/jruby.jar"]).join(':')
+  system %(javac -Xlint:-options -source 1.6 -target 1.6 -cp #{classpath} ext/java/*.java ext/java/org/msgpack/jruby/*.java)
+  exit($?.exitstatus) unless $?.success?
 end
 
-BENCHMARK_RUBIES = ['1.9.2-p290', 'jruby-1.6.2', 'jruby-1.6.4']
-BENCHMARK_GEMSET = 'msgpack-jruby-benchmarking'
+task :package => :compile do
+  class_files = Dir['ext/java/**/*.class'].map { |path| path = path.sub('ext/java/', ''); "-C ext/java '#{path}'" }
+  system %(jar cf lib/ext/msgpack_jruby.jar #{class_files.join(' ')})
+  exit($?.exitstatus) unless $?.success?
+end
 
-task :benchmark do
-  rubies = BENCHMARK_RUBIES.map { |rb| "#{rb}@#{BENCHMARK_GEMSET}" }
-  cmd = %(rvm #{rubies.join(',')} exec viiite run spec/benchmark.rb | tee benchmark | viiite report --hierarchy --regroup=bench,ruby)
-  puts cmd
-  system cmd
+task :release do
+  version_string = "v#{MessagePack::VERSION}"
+  unless %x(git tag -l).include?(version_string)
+    system %(git tag -a #{version_string} -m #{version_string})
+  end
+  system %(gem build msgpack-jruby.gemspec && gem push msgpack-jruby-*.gem && mv msgpack-jruby-*.gem pkg)
 end
+
+namespace :benchmark do
+  BENCHMARK_RUBIES = ['1.9.2-p0', 'jruby-1.6.5', 'jruby-head']
+  BENCHMARK_GEMSET = 'msgpack-jruby-benchmarking'
+
+  task :run do
+    rubies = BENCHMARK_RUBIES.map { |rb| "#{rb}@#{BENCHMARK_GEMSET}" }
+    cmd = %(rvm #{rubies.join(',')} exec viiite run spec/benchmark.rb | tee benchmark | viiite report --hierarchy --regroup=bench,lib,ruby)
+    puts cmd
+    system cmd
+  end
+
+  task :quick do
+    cmd = %(IMPLEMENTATIONS=msgpack viiite run spec/benchmark.rb | viiite report --hierarchy --regroup=bench)
+    puts cmd
+    system cmd
+  end
+
+  task :setup do
+    rubies = BENCHMARK_RUBIES.map { |rb| "#{rb}@#{BENCHMARK_GEMSET}" }
+    rubies.each do |ruby_version|
+      cmd = %(rvm-shell #{ruby_version} -c 'bundle check || bundle install')
+      puts cmd
+      system cmd
+    end
+  end
+end

data/ext/.gitignore

@@ -0,0 +1 @@
+*.class

data/ext/java/MsgpackJrubyService.java

@@ -0,0 +1,15 @@
+import java.io.IOException;
+        
+import org.jruby.Ruby;
+import org.jruby.runtime.load.BasicLibraryService;
+
+import org.msgpack.jruby.MessagePackLibrary;
+
+
+public class MsgpackJrubyService implements BasicLibraryService {
+  public boolean basicLoad(final Ruby runtime) throws IOException {
+    new MessagePackLibrary().load(runtime, false);
+    return true;
+  }
+}
+

data/ext/java/org/msgpack/jruby/MessagePackLibrary.java

@@ -0,0 +1,229 @@
+package org.msgpack.jruby;
+
+
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import org.jruby.Ruby;
+import org.jruby.RubyModule;
+import org.jruby.RubyClass;
+import org.jruby.RubyString;
+import org.jruby.RubyObject;
+import org.jruby.RubyIO;
+import org.jruby.RubyStringIO;
+import org.jruby.RubyNumeric;
+import org.jruby.RubyEnumerator;
+import org.jruby.runtime.load.Library;
+import org.jruby.runtime.callback.Callback;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.runtime.Arity;
+import org.jruby.runtime.Block;
+import org.jruby.runtime.ObjectAllocator;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.anno.JRubyClass;
+import org.jruby.anno.JRubyModule;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.util.IOInputStream;
+
+import static org.jruby.runtime.Visibility.*;
+
+import org.msgpack.MessagePack;
+import org.msgpack.packer.BufferPacker;
+import org.msgpack.packer.Packer;
+import org.msgpack.unpacker.MessagePackBufferUnpacker;
+import org.msgpack.unpacker.MessagePackUnpacker;
+import org.msgpack.unpacker.UnpackerIterator;
+import org.msgpack.type.Value;
+import org.msgpack.io.Input;
+import org.msgpack.io.LinkedBufferInput;
+import org.msgpack.io.StreamInput;
+
+
+public class MessagePackLibrary implements Library {
+  public void load(Ruby runtime, boolean wrap) throws IOException {
+    MessagePack msgPack = new MessagePack();
+    RubyModule msgpackModule = runtime.defineModule("MessagePack");
+    msgpackModule.defineAnnotatedMethods(MessagePackModule.class);
+    RubyClass standardErrorClass = runtime.getStandardError();
+    RubyClass unpackErrorClass = msgpackModule.defineClassUnder("UnpackError", standardErrorClass, standardErrorClass.getAllocator());
+    RubyClass unpackerClass = msgpackModule.defineClassUnder("Unpacker", runtime.getObject(), new UnpackerAllocator(msgPack));
+    unpackerClass.defineAnnotatedMethods(Unpacker.class);
+  }
+
+  @JRubyModule(name = "MessagePack")
+  public static class MessagePackModule {
+    private static MessagePack msgPack = new MessagePack();
+    private static RubyObjectUnpacker unpacker = new RubyObjectUnpacker(msgPack);
+    
+    @JRubyMethod(module = true, required = 1)
+    public static IRubyObject pack(ThreadContext ctx, IRubyObject recv, IRubyObject obj) throws IOException {
+      BufferPacker bufferedPacker = msgPack.createBufferPacker();
+      Packer packer = new RubyObjectPacker(msgPack, bufferedPacker).write(obj);
+      return RubyString.newString(ctx.getRuntime(), bufferedPacker.toByteArray());
+    }
+    
+    @JRubyMethod(module = true, required = 1)
+    public static IRubyObject unpack(ThreadContext ctx, IRubyObject recv, IRubyObject obj) throws IOException {
+      return unpacker.unpack(obj.asString());
+    }
+  }
+
+  private static class UnpackerAllocator implements ObjectAllocator {
+    private MessagePack msgPack;
+      
+    public UnpackerAllocator(MessagePack msgPack) {
+      this.msgPack = msgPack;
+    }
+      
+    public IRubyObject allocate(Ruby runtime, RubyClass klass) {
+      return new Unpacker(runtime, klass, msgPack);
+    }
+  }
+
+  @JRubyClass(name="MessagePack::Unpacker")
+  public static class Unpacker extends RubyObject {
+    private MessagePack msgPack;
+    private RubyObjectUnpacker rubyObjectUnpacker;
+    private MessagePackBufferUnpacker bufferUnpacker;
+    private MessagePackUnpacker streamUnpacker;
+    private IRubyObject stream;
+    private IRubyObject data;
+    
+    public Unpacker(Ruby runtime, RubyClass type, MessagePack msgPack) {
+      super(runtime, type);
+      this.msgPack = msgPack;
+      this.rubyObjectUnpacker = new RubyObjectUnpacker(msgPack);
+      this.bufferUnpacker = null;
+      this.streamUnpacker = null;
+      this.stream = null;
+      this.data = null;
+    }
+
+    @JRubyMethod(name = "initialize", optional = 1, visibility = PRIVATE)
+    public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) {
+      if (args.length > 0) {
+        setStream(ctx, args[0]);
+      }
+      return this;
+    }
+
+    @JRubyMethod(required = 2)
+    public IRubyObject execute(ThreadContext ctx, IRubyObject data, IRubyObject offset) {
+      return executeLimit(ctx, data, offset, null);
+    }
+
+    @JRubyMethod(name = "execute_limit", required = 3)
+    public IRubyObject executeLimit(ThreadContext ctx, IRubyObject data, IRubyObject offset, IRubyObject limit) {
+      this.data = null;
+      try {
+        int jOffset = RubyNumeric.fix2int(offset);
+        int jLimit = -1;
+        if (limit != null) {
+          jLimit = RubyNumeric.fix2int(limit);
+        }
+        byte[] bytes = data.asString().getBytes();
+        MessagePackBufferUnpacker localBufferUnpacker = new MessagePackBufferUnpacker(msgPack, bytes.length);
+        localBufferUnpacker.wrap(bytes, jOffset, jLimit == -1 ? bytes.length - jOffset : jLimit);
+        this.data = rubyObjectUnpacker.valueToRubyObject(ctx.getRuntime(), localBufferUnpacker.readValue());
+        return ctx.getRuntime().newFixnum(jOffset + localBufferUnpacker.getReadByteCount());
+      } catch (IOException ioe) {
+        // TODO: how to throw Ruby exceptions?
+        return ctx.getRuntime().getNil();
+      }
+    }
+
+    @JRubyMethod(name = "data")
+    public IRubyObject getData(ThreadContext ctx) {
+      if (data == null) {
+        return ctx.getRuntime().getNil();
+      } else {
+        return data;
+      }
+    }
+
+    @JRubyMethod(name = "finished?")
+    public IRubyObject finished_q(ThreadContext ctx) {
+      return data == null ? ctx.getRuntime().getFalse() : ctx.getRuntime().getTrue();
+    }
+
+    @JRubyMethod(required = 1)
+    public IRubyObject feed(ThreadContext ctx, IRubyObject data) {
+      streamUnpacker = null;
+      byte[] bytes = data.asString().getBytes();
+      if (bufferUnpacker == null) {
+        bufferUnpacker = new MessagePackBufferUnpacker(msgPack);
+      }
+      bufferUnpacker.feed(bytes);
+      return ctx.getRuntime().getNil();
+    }
+
+    @JRubyMethod(name = "feed_each", required = 1)
+    public IRubyObject feedEach(ThreadContext ctx, IRubyObject data, Block block) {
+      feed(ctx, data);
+      each(ctx, block);
+      return ctx.getRuntime().getNil();
+    }
+    
+    @JRubyMethod
+    public IRubyObject each(ThreadContext ctx, Block block) {
+      MessagePackUnpacker localUnpacker = null;
+      if (bufferUnpacker == null && streamUnpacker != null) {
+        localUnpacker = streamUnpacker;
+      } else if (bufferUnpacker != null) {
+        localUnpacker = bufferUnpacker;
+      } else {
+        return ctx.getRuntime().getNil();
+      }
+      if (block.isGiven()) {
+        for (Value value : localUnpacker) {
+          IRubyObject rubyObject = rubyObjectUnpacker.valueToRubyObject(ctx.getRuntime(), value);
+          block.yield(ctx, rubyObject);
+        }
+        return ctx.getRuntime().getNil();
+      } else {
+        return RubyEnumerator.RubyEnumeratorKernel.obj_to_enum(ctx, this, Block.NULL_BLOCK);
+      }
+    }
+
+    @JRubyMethod
+    public IRubyObject fill(ThreadContext ctx) {
+      return ctx.getRuntime().getNil();
+    }
+
+    @JRubyMethod
+    public IRubyObject reset(ThreadContext ctx) {
+      if (bufferUnpacker != null) {
+        bufferUnpacker.reset();
+      }
+      if (streamUnpacker != null) {
+        streamUnpacker.reset();
+      }
+      return ctx.getRuntime().getNil();
+    }
+
+    @JRubyMethod(name = "stream")
+    public IRubyObject getStream(ThreadContext ctx) {
+      if (stream == null) {
+        return ctx.getRuntime().getNil();
+      } else {
+        return stream;
+      }
+    }
+
+    @JRubyMethod(name = "stream=", required = 1)
+    public IRubyObject setStream(ThreadContext ctx, IRubyObject stream) {
+      bufferUnpacker = null;
+      this.stream = stream;
+      if (stream instanceof RubyStringIO) {
+        // TODO: RubyStringIO returns negative numbers when read through IOInputStream#read
+        IRubyObject str = ((RubyStringIO) stream).string();
+        byte[] bytes = ((RubyString) str).getBytes();
+        streamUnpacker = new MessagePackUnpacker(msgPack, new ByteArrayInputStream(bytes));
+      } else {
+        streamUnpacker = new MessagePackUnpacker(msgPack, new IOInputStream(stream));
+      }
+      return getStream(ctx);
+    }
+  }
+}

data/ext/java/org/msgpack/jruby/RubyObjectPacker.java

@@ -0,0 +1,111 @@
+package org.msgpack.jruby;
+
+
+import java.io.IOException;
+
+import org.msgpack.MessagePack;
+import org.msgpack.packer.Packer;
+import org.msgpack.packer.MessagePackBufferPacker;
+
+import org.jruby.RubyObject;
+import org.jruby.RubyNil;
+import org.jruby.RubyBoolean;
+import org.jruby.RubyBignum;
+import org.jruby.RubyInteger;
+import org.jruby.RubyFixnum;
+import org.jruby.RubyFloat;
+import org.jruby.RubyString;
+import org.jruby.RubySymbol;
+import org.jruby.RubyArray;
+import org.jruby.RubyHash;
+import org.jruby.runtime.builtin.IRubyObject;
+
+
+public class RubyObjectPacker extends MessagePackBufferPacker {
+  private final Packer packer;
+
+  public RubyObjectPacker(MessagePack msgPack, Packer packer) {
+    super(msgPack);
+    this.packer = packer;
+  }
+
+  public Packer write(IRubyObject o) throws IOException {
+    if (o == null || o instanceof RubyNil) {
+      packer.writeNil();
+      return this;
+    } else if (o instanceof RubyBoolean) {
+      packer.write(((RubyBoolean) o).isTrue());
+      return this;
+    } else if (o instanceof RubyBignum) {
+      return write((RubyBignum) o);
+    } else if (o instanceof RubyInteger) {
+      return write((RubyInteger) o);
+    } else if (o instanceof RubyFixnum) {
+      return write((RubyFixnum) o);
+    } else if (o instanceof RubyFloat) {
+      return write((RubyFloat) o);
+    } else if (o instanceof RubyString) {
+      return write((RubyString) o);
+    } else if (o instanceof RubySymbol) {
+      return write((RubySymbol) o);
+    } else if (o instanceof RubyArray) {
+      return write((RubyArray) o);
+    } else if (o instanceof RubyHash) {
+      return write((RubyHash) o);
+    } else {
+      throw o.getRuntime().newArgumentError(String.format("Cannot pack type: %s", o.getClass().getName()));
+    }
+  }
+
+  public final Packer write(RubyBignum bignum) throws IOException {
+    packer.write(bignum.getBigIntegerValue());
+    return this;
+  }
+
+  public final Packer write(RubyInteger integer) throws IOException {
+    packer.write(integer.getLongValue());
+    return this;
+  }
+
+  public final Packer write(RubyFixnum fixnum) throws IOException {
+    packer.write(fixnum.getLongValue());
+    return this;
+  }
+
+  public final Packer write(RubyFloat flt) throws IOException {
+    packer.write(flt.getDoubleValue());
+    return this;
+  }
+
+  public final Packer write(RubyString str) throws IOException {
+    packer.write(str.getBytes());
+    return this;
+  }
+
+  public final Packer write(RubySymbol sym) throws IOException {
+    return write((RubyString) sym.to_s());
+  }
+
+  public final Packer write(RubyArray array) throws IOException {
+    int count = array.size();
+    packer.writeArrayBegin(count);
+    for (int i = 0; i < count; i++) {
+      write((RubyObject) array.entry(i));
+    }
+    packer.writeArrayEnd();
+    return this;
+  }
+
+  public final Packer write(RubyHash hash) throws IOException {
+    int count = hash.size();
+    packer.writeMapBegin(count);
+    RubyArray keys = hash.keys();
+    RubyArray values = hash.rb_values();
+    for (int i = 0; i < count; i++) {
+      write((RubyObject) keys.entry(i));
+      write((RubyObject) values.entry(i));
+    }
+    packer.writeMapEnd();
+    return this;
+  }
+}

data/ext/java/org/msgpack/jruby/RubyObjectUnpacker.java

@@ -0,0 +1,114 @@
+package org.msgpack.jruby;
+
+
+import java.io.IOException;
+
+import org.msgpack.MessagePack;
+import org.msgpack.MessageTypeException;
+import org.msgpack.unpacker.MessagePackBufferUnpacker;
+import org.msgpack.type.Value;
+import org.msgpack.type.ValueType;
+import org.msgpack.type.BooleanValue;
+import org.msgpack.type.IntegerValue;
+import org.msgpack.type.FloatValue;
+import org.msgpack.type.ArrayValue;
+import org.msgpack.type.MapValue;
+import org.msgpack.type.RawValue;
+
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.RubyNil;
+import org.jruby.RubyBoolean;
+import org.jruby.RubyBignum;
+import org.jruby.RubyInteger;
+import org.jruby.RubyFixnum;
+import org.jruby.RubyFloat;
+import org.jruby.RubyString;
+import org.jruby.RubySymbol;
+import org.jruby.RubyArray;
+import org.jruby.RubyHash;
+import org.jruby.runtime.builtin.IRubyObject;
+
+
+public class RubyObjectUnpacker {
+  private final MessagePack msgPack;
+
+  public RubyObjectUnpacker(MessagePack msgPack) {
+    this.msgPack = msgPack;
+  }
+
+  public IRubyObject unpack(RubyString str) throws IOException {
+    return unpack(str.getRuntime(), str.getBytes());
+  }
+
+  public IRubyObject unpack(Ruby runtime, byte[] data) throws IOException {
+    MessagePackBufferUnpacker unpacker = new MessagePackBufferUnpacker(msgPack);
+    unpacker.wrap(data);
+    return valueToRubyObject(runtime, unpacker.readValue()); 
+  }
+
+  IRubyObject valueToRubyObject(Ruby runtime, Value value) {
+    switch (value.getType()) {
+    case NIL:
+      return runtime.getNil();
+    case BOOLEAN:
+      return convert(runtime, value.asBooleanValue());
+    case INTEGER:
+      return convert(runtime, value.asIntegerValue());
+    case FLOAT:
+      return convert(runtime, value.asFloatValue());
+    case ARRAY:
+      return convert(runtime, value.asArrayValue());
+    case MAP:
+      return convert(runtime, value.asMapValue());
+    case RAW:
+      return convert(runtime, value.asRawValue());
+    default:
+      throw runtime.newArgumentError(String.format("Unexpected value: %s", value.toString()));
+    }
+  }
+
+  private IRubyObject convert(Ruby runtime, BooleanValue value) {
+    return RubyBoolean.newBoolean(runtime, value.asBooleanValue().getBoolean());
+  }
+
+  private IRubyObject convert(Ruby runtime, IntegerValue value) {
+    // TODO: is there any way of checking for bignums up front?
+    IntegerValue iv = value.asIntegerValue();
+    try {
+      return RubyFixnum.newFixnum(runtime, iv.getLong());
+    } catch (MessageTypeException mte) {
+      return RubyBignum.newBignum(runtime, iv.getBigInteger());
+    }
+  }
+
+  private IRubyObject convert(Ruby runtime, FloatValue value) {
+    return RubyFloat.newFloat(runtime, value.asFloatValue().getDouble());
+  }
+
+  private IRubyObject convert(Ruby runtime, ArrayValue value) {
+    Value[] elements = value.asArrayValue().getElementArray();
+    int elementCount = elements.length;
+    IRubyObject[] rubyObjects = new IRubyObject[elementCount];
+    for (int i = 0; i < elementCount; i++) {
+      rubyObjects[i] = valueToRubyObject(runtime, elements[i]);
+    }
+    return RubyArray.newArray(runtime, rubyObjects);
+  }
+
+  private IRubyObject convert(Ruby runtime, MapValue value) {
+    Value[] keysAndValues = value.asMapValue().getKeyValueArray();
+    int kvCount = keysAndValues.length;
+    RubyHash hash = RubyHash.newHash(runtime);
+    for (int i = 0; i < kvCount; i += 2) {
+      Value k = keysAndValues[i];
+      Value v = keysAndValues[i + 1];
+      hash.put(valueToRubyObject(runtime, k), valueToRubyObject(runtime, v));
+    }
+    return hash;
+  }
+
+  private IRubyObject convert(Ruby runtime, RawValue value) {
+    return RubyString.newString(runtime, value.asRawValue().getByteArray());
+  }
+}

data/lib/msgpack/version.rb

@@ -1,3 +1,3 @@
 module MessagePack
-  VERSION = '1.0.0'
+  VERSION = '1.1.3'
 end

data/lib/msgpack.rb

@@ -1,7 +1,8 @@
 # encoding: utf-8
 
-require 'java'
-
-$CLASSPATH << File.expand_path('../ext', __FILE__) << File.expand_path('../ext/msgpack-0.5.2-devel.jar', __FILE__)
+$: << File.expand_path('../ext', __FILE__)
 
-MessagePack = org.msgpack.jruby.MessagePack
+require 'java'
+require 'javassist-3.15.0-GA'
+require 'msgpack-0.6.6'
+require 'msgpack_jruby'

data/spec/benchmark.rb

@@ -10,17 +10,23 @@ require 'msgpack'
 require 'json'
 require 'bson'
 
+if RUBY_PLATFORM.include?('java')
+  BSON_IMPL = BSON::BSON_JAVA
+else
+  BSON_IMPL = BSON::BSON_C
+end
 
 OBJECT_STRUCTURE = {'x' => ['y', 34, 2**30 + 3, 2.1223423423356, {'hello' => 'world', '5' => [63, 'asjdl']}]}
 ENCODED_MSGPACK = "\x81\xA1x\x95\xA1y\"\xCE@\x00\x00\x03\xCB@\x00\xFA\x8E\x9F9\xFA\xC1\x82\xA5hello\xA5world\xA15\x92?\xA5asjdl"
 ENCODED_BSON = "d\x00\x00\x00\x04x\x00\\\x00\x00\x00\x020\x00\x02\x00\x00\x00y\x00\x101\x00\"\x00\x00\x00\x102\x00\x03\x00\x00@\x013\x00\xC1\xFA9\x9F\x8E\xFA\x00@\x034\x002\x00\x00\x00\x02hello\x00\x06\x00\x00\x00world\x00\x045\x00\x19\x00\x00\x00\x100\x00?\x00\x00\x00\x021\x00\x06\x00\x00\x00asjdl\x00\x00\x00\x00\x00"
 ENCODED_JSON = '{"x":["y",34,1073741827,2.1223423423356,{"hello":"world","5":[63,"asjdl"]}]}'
 ITERATIONS = 1_00_000
+IMPLEMENTATIONS = ENV.fetch('IMPLEMENTATIONS', 'json,bson,msgpack').split(',').map(&:to_sym)
 
 Viiite.bm do |b|
   b.variation_point :ruby, Viiite.which_ruby
   
-  [:json, :msgpack].each do |lib|
+  IMPLEMENTATIONS.each do |lib|
     b.variation_point :lib, lib
     
   
@@ -28,7 +34,7 @@ Viiite.bm do |b|
       ITERATIONS.times do
         case lib
         when :msgpack then MessagePack.pack(OBJECT_STRUCTURE)
-        when :bson then BSON::BSON_JAVA.serialize(OBJECT_STRUCTURE).to_s
+        when :bson then BSON_IMPL.serialize(OBJECT_STRUCTURE).to_s
         when :json then OBJECT_STRUCTURE.to_json
         end
       end
@@ -38,7 +44,7 @@ Viiite.bm do |b|
       ITERATIONS.times do
         case lib
         when :msgpack then MessagePack.unpack(ENCODED_MSGPACK)
-        when :bson then BSON::BSON_JAVA.deserialize(ENCODED_BSON)
+        when :bson then BSON_IMPL.deserialize(ENCODED_BSON)
         when :json then JSON.parse(ENCODED_JSON)
         end
       end
@@ -48,7 +54,7 @@ Viiite.bm do |b|
       ITERATIONS.times do
         case lib
         when :msgpack then MessagePack.unpack(MessagePack.pack(OBJECT_STRUCTURE))
-        when :bson then BSON::BSON_JAVA.deserialize(BSON::BSON_JAVA.serialize(OBJECT_STRUCTURE).to_s)
+        when :bson then BSON_IMPL.deserialize(BSON_IMPL.serialize(OBJECT_STRUCTURE).to_s)
         when :json then JSON.parse(OBJECT_STRUCTURE.to_json)
         end
       end
@@ -58,7 +64,7 @@ Viiite.bm do |b|
       ITERATIONS.times do
         case lib
         when :msgpack then MessagePack.pack(MessagePack.unpack(ENCODED_MSGPACK))
-        when :bson then BSON::BSON_JAVA.serialize(BSON::BSON_JAVA.deserialize(ENCODED_BSON)).to_s
+        when :bson then BSON_IMPL.serialize(BSON_IMPL.deserialize(ENCODED_BSON)).to_s
         when :json then OBJECT_STRUCTURE.to_json(JSON.parse(ENCODED_JSON))
         end
       end

data/spec/msgpack/unpacker_spec.rb

@@ -0,0 +1,159 @@
+# encoding: ascii-8bit
+
+require_relative '../spec_helper'
+require 'stringio'
+require 'tempfile'
+
+
+describe ::MessagePack::Unpacker do
+  subject do
+    described_class.new
+  end
+  
+  let :buffer1 do
+    MessagePack.pack(:foo => 'bar')
+  end
+    
+  let :buffer2 do
+    MessagePack.pack(:hello => {:world => [1, 2, 3]})
+  end
+    
+  let :buffer3 do
+    MessagePack.pack(:x => 'y')
+  end
+  
+  describe '#execute/#execute_limit/#finished?' do
+    let :buffer do
+      buffer1 + buffer2 + buffer3
+    end
+    
+    it 'extracts an object from the buffer' do
+      subject.execute(buffer, 0)
+      subject.data.should == {'foo' => 'bar'}
+    end
+
+    it 'extracts an object from the buffer, starting at an offset' do
+      subject.execute(buffer, buffer1.length)
+      subject.data.should == {'hello' => {'world' => [1, 2, 3]}}
+    end
+
+    it 'extracts an object from the buffer, starting at an offset reading bytes up to a limit' do
+      subject.execute_limit(buffer, buffer1.length, buffer2.length)
+      subject.data.should == {'hello' => {'world' => [1, 2, 3]}}
+    end
+    
+    it 'extracts nothing if the limit cuts an object in half' do
+      subject.execute_limit(buffer, buffer1.length, 3)
+      subject.data.should be_nil
+    end
+
+    it 'returns the offset where the object ended' do
+      subject.execute(buffer, 0).should == buffer1.length
+      subject.execute(buffer, buffer1.length).should == buffer1.length + buffer2.length
+    end
+    
+    it 'is finished if #data returns an object' do
+      subject.execute_limit(buffer, buffer1.length, buffer2.length)
+      subject.should be_finished
+      
+      subject.execute_limit(buffer, buffer1.length, 3)
+      subject.should_not be_finished
+    end
+  end
+  
+  describe '#each' do
+    context 'with a buffer' do
+      it 'yields each object in the buffer' do
+        objects = []
+        subject.feed(buffer1)
+        subject.feed(buffer2)
+        subject.feed(buffer3)
+        subject.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+      
+      it 'returns an enumerator when no block is given' do
+        subject.feed(buffer1)
+        subject.feed(buffer2)
+        subject.feed(buffer3)
+        enum = subject.each
+        enum.map { |obj| obj.keys.first }.should == %w[foo hello x]
+      end
+    end
+    
+    context 'with a StringIO stream' do
+      it 'yields each object in the stream' do
+        objects = []
+        subject.stream = StringIO.new(buffer1 + buffer2 + buffer3)
+        subject.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+    end
+    
+    context 'with a File stream' do
+      it 'yields each object in the stream' do
+        objects = []
+        file = Tempfile.new('msgpack')
+        file.write(buffer1)
+        file.write(buffer2)
+        file.write(buffer3)
+        file.open
+        subject.stream = file
+        subject.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+    end
+    
+    context 'with a stream passed to the constructor' do
+      it 'yields each object in the stream' do
+        objects = []
+        unpacker = described_class.new(StringIO.new(buffer1 + buffer2 + buffer3))
+        unpacker.each do |obj|
+          objects << obj
+        end
+        objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+      end
+    end
+  end
+  
+  describe '#feed_each' do
+    it 'feeds the buffer then runs #each' do
+      objects = []
+      subject.feed_each(buffer1 + buffer2 + buffer3) do |obj|
+        objects << obj
+      end
+      objects.should == [{'foo' => 'bar'}, {'hello' => {'world' => [1, 2, 3]}}, {'x' => 'y'}]
+    end
+  end
+  
+  describe '#fill' do
+    it 'is a no-op' do
+      subject.stream = StringIO.new(buffer1 + buffer2 + buffer3)
+      subject.fill
+      subject.each { |obj| }
+    end
+  end
+  
+  describe '#reset' do
+    context 'with a buffer' do
+      it 'is unclear what it is supposed to do'
+    end
+    
+    context 'with a stream' do
+      it 'is unclear what it is supposed to do'
+    end
+  end
+
+  context 'regressions' do
+    it 'handles massive arrays (issue #2)' do
+      array = ['foo'] * 10_000
+      MessagePack.unpack(MessagePack.pack(array)).should have(10_000).items
+    end
+  end
+end

data/spec/msgpack_spec.rb

@@ -69,8 +69,13 @@ describe MessagePack do
   end
 
   context 'with other things' do
-    it 'raises an error on #pack' do
+    it 'raises an error on #pack with an unsupported type' do
       expect { MessagePack.pack(self) }.to raise_error(ArgumentError, /^Cannot pack type:/)
     end
+    
+    it 'rasies an error on #unpack with garbage' do
+      pending
+      expect { MessagePack.unpack('asdka;sd') }.to raise_error(MessagePack::UnpackError)
+    end
   end
 end

metadata

@@ -1,76 +1,70 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: msgpack-jruby
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 0
-  - 0
-  version: 1.0.0
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 1.1.3
 platform: java
-authors: 
+authors:
 - Theo Hultberg
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2011-09-15 00:00:00 +02:00
-default_executable: 
+date: 2012-06-14 00:00:00.000000000Z
 dependencies: []
-
 description: JRuby compatible MessagePack implementation that does not use FFI
-email: 
+email:
 - theo@iconara.net
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - .rspec
 - .rvmrc
 - Gemfile
+- Gemfile.lock
 - Rakefile
-- ext/java/org/msgpack/jruby/MessagePack.java
-- lib/ext/msgpack-0.5.2-devel.jar
-- lib/ext/org/msgpack/jruby/MessagePack.class
+- ext/.gitignore
+- ext/java/MsgpackJrubyService.java
+- ext/java/org/msgpack/jruby/MessagePackLibrary.java
+- ext/java/org/msgpack/jruby/RubyObjectPacker.java
+- ext/java/org/msgpack/jruby/RubyObjectUnpacker.java
+- lib/ext/javassist-3.15.0-GA.jar
+- lib/ext/msgpack-0.6.6.jar
+- lib/ext/msgpack_jruby.jar
 - lib/msgpack.rb
 - lib/msgpack/version.rb
 - msgpack-jruby.gemspec
 - spec/benchmark.rb
+- spec/msgpack/unpacker_spec.rb
 - spec/msgpack_spec.rb
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: http://github.com/iconara/msgpack-jruby
 licenses: []
-
-post_install_message: 
+post_install_message:
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+      hash: 2
+      version: '0'
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
 requirements: []
-
 rubyforge_project: msgpack-jruby
-rubygems_version: 1.3.6
-signing_key: 
+rubygems_version: 1.8.15
+signing_key:
 specification_version: 3
 summary: MessagePack implementation for JRuby
 test_files: []
-
+...

data/ext/java/org/msgpack/jruby/MessagePack.java

@@ -1,156 +0,0 @@
-package org.msgpack.jruby;
-
-
-import java.io.IOException;
-import java.io.ByteArrayOutputStream;
-import java.math.BigInteger;
-import java.util.Map;
-import java.util.HashMap;
-import java.lang.reflect.Field;
-import org.jruby.runtime.builtin.IRubyObject;
-import org.jruby.Ruby;
-import org.jruby.RubyObject;
-import org.jruby.RubyString;
-import org.jruby.RubySymbol;
-import org.jruby.RubyArray;
-import org.jruby.RubyHash;
-import org.jruby.RubyBoolean;
-import org.jruby.RubyInteger;
-import org.jruby.RubyFixnum;
-import org.jruby.RubyFloat;
-import org.jruby.RubyBignum;
-import org.jruby.RubyNil;
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-import org.msgpack.MessagePackObject;
-import org.msgpack.MessageTypeException;
-import org.msgpack.object.MapType;
-
-
-public class MessagePack {
-	private static final MessagePack instance = new MessagePack();
-	
-	public static RubyString pack(RubyObject o) throws IOException {
-		return instance.packObject(o);
-	}
-	
-	public static RubyObject unpack(RubyString s) throws IOException {
-		return instance.unpackString(s);
-	}
-	
-	public RubyString packObject(IRubyObject o) throws IOException {
-		ByteArrayOutputStream stream = new ByteArrayOutputStream();
-		_pack(o, new Packer(stream));
-		Ruby runtime = o == null ? Ruby.getGlobalRuntime() : o.getRuntime();
-		return RubyString.newString(runtime, stream.toByteArray());
-	}
-	
-	private void _pack(IRubyObject o, Packer packer) throws IOException {
-		if (o == null || o instanceof RubyNil) {
-			packer.packNil();
-		} else if (o instanceof RubyBoolean) {
-			packer.packBoolean((Boolean) ((RubyBoolean) o).toJava(Boolean.class));
-		} else if (o instanceof RubyBignum) {
-			packer.packBigInteger(((RubyBignum) o).getBigIntegerValue());
-		} else if (o instanceof RubyInteger) {
-			packer.packLong(((RubyInteger) o).getLongValue());
-		} else if (o instanceof RubyFixnum) {
-			packer.packLong(((RubyFixnum) o).getLongValue());
-		} else if (o instanceof RubyFloat) {
-			packer.packDouble(((RubyFloat) o).getDoubleValue());
-		} else if (o instanceof RubyString) {
-			byte[] bytes = ((RubyString) o).getBytes();
-			packer.packRaw(bytes.length);
-			packer.packRawBody(bytes);
-		} else if (o instanceof RubySymbol) {
-			_pack((IRubyObject) ((RubySymbol) o).to_s(), packer);
-		} else if (o instanceof RubyArray) {
-			RubyArray array = (RubyArray) o;
-			int count = array.size();
-			packer.packArray(count);
-			for (int i = 0; i < count; i++) {
-				_pack(array.entry(i), packer);
-			}
-		} else if (o instanceof RubyHash) {
-			RubyHash hash = (RubyHash) o;
-			int count = hash.size();
-			packer.packMap(count);
-			RubyArray keys = hash.keys();
-			RubyArray values = hash.rb_values();
-			for (int i = 0; i < count; i++) {
-				_pack(keys.entry(i), packer);
-				_pack(values.entry(i), packer);
-			}
-		} else {
-			throw Ruby.getGlobalRuntime().newArgumentError(String.format("Cannot pack type: %s", o.getClass().getName()));
-		}
-	}
-
-	public RubyObject unpackString(RubyString blob) throws IOException {
-		RubyObject obj = null;
-		Unpacker unpacker = new Unpacker();
-		unpacker.wrap(blob.getBytes());
-		return _unpack(unpacker.iterator().next(), blob.getRuntime());
-	}
-	
-	private static Field cachedMapField = null;
-	
-	private RubyObject _unpack(MessagePackObject mpo, Ruby runtime) throws IOException {
-		if (mpo.isNil()) {
-			return null;
-		} else if (mpo.isBooleanType()) {
-			return RubyBoolean.newBoolean(runtime, mpo.asBoolean());
-		} else if (mpo.isIntegerType()) {
-			try {
-				return RubyFixnum.newFixnum(runtime, mpo.asLong());
-			} catch (MessageTypeException mte) {
-				return RubyBignum.newBignum(runtime, mpo.asBigInteger());
-			}
-		} else if (mpo.isFloatType()) {
-			return RubyFloat.newFloat(runtime, mpo.asDouble());
-		} else if (mpo.isArrayType()) {
-			MessagePackObject[] mpoElements = mpo.asArray();
-			RubyObject[] elements = new RubyObject[mpoElements.length];
-			int count = mpoElements.length;
-			for (int i = 0; i < count; i++ ) {
-				elements[i] = _unpack(mpoElements[i], runtime);
-			}
-			return RubyArray.newArray(runtime, elements);
-		} else if (mpo.isMapType()) {
-			// This is hackish, but the only way to make sure that hashes 
-			// keep their order. MessagePack's unpacking subsystem does not 
-			// expose the key/value pair array, only a Map constructed from
-			// that array. This code first attempts to get that array from a
-			// private field, and if that fails it falls back on using the
-			// unordered Map.
-			RubyHash hash = null;
-			try {
-				if (cachedMapField == null) {
-					cachedMapField = mpo.getClass().getDeclaredField("map");
-					cachedMapField.setAccessible(true);
-				}
-				MessagePackObject[] keyValuePairs = (MessagePackObject[]) cachedMapField.get(mpo);
-				int count = keyValuePairs.length;
-				hash = RubyHash.newHash(runtime);
-				for (int i = 0; i < count; i += 2) {
-					hash.put(_unpack(keyValuePairs[i], runtime), _unpack(keyValuePairs[i + 1], runtime));
-				}
-			} catch (IllegalAccessException iae) {
-			} catch (NoSuchFieldException nfe) {
-			}
-			if (hash == null) {
-				Map<MessagePackObject, MessagePackObject> mpoMap = mpo.asMap();
-				hash = RubyHash.newHash(runtime);
-				for (Map.Entry<MessagePackObject, MessagePackObject> entry : mpoMap.entrySet()) {
-					hash.put(_unpack(entry.getKey(), runtime), _unpack(entry.getValue(), runtime));
-				}
-			}
-			return hash;
-			
-		} else if (mpo.isRawType()) {
-			return RubyString.newString(runtime, mpo.asByteArray());
-		} else {
-			throw runtime.newArgumentError(String.format("Cannot upack type: %s", mpo.getClass().getName()));
-		}
-	}
-}