msgpack 0.7.0dev1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e934ddd7840719b64ca248c14b5cb3e81ed4c0fd
-  data.tar.gz: 2bd6ef14c3c873241be2911fe0666dc5526e9351
+  metadata.gz: 08b929d16e3f5eb48b0de6a10041092be2510361
+  data.tar.gz: 34c606ea1c703c22f94d1295b5648f4430fd69ae
 SHA512:
-  metadata.gz: 857b591765e1b4bb0a0efac427e729e278168eee8bdfdd47bc9b1a86f30edfdecedc4a81626709da5d21fb25988485c243532d3d87d9c3cde1b9a47c30c2624b
-  data.tar.gz: a616c639a3c3cc8029d1bc93d8821f60ad85799995016e7cf579a0875380de21b598a0dbad608db728fc8301ce993453db1d7605ae189e51af2aa04f2e712624
+  metadata.gz: 4ec69a5b4e904359b7c1d1a15f37ee2e57fa5e340fbab0e658cf2d476d736d41100cd350ce580f52afbbb88721640be3bd120583adfd6f179aa1b3afa4395142
+  data.tar.gz: 63bf4dd13deaa7c767bbc1f6c50170548cb6819ebb2ca87282939f6cd88da078820af2453634dcb0e4c3d056073807f34ac86623c5b75ec89ea416ed8fa6f7c1

data/.gitignore

@@ -11,6 +11,7 @@ Gemfile*
 pkg
 test/debug.log
 *~
+*.swp
 /rdoc
 tmp
 .classpath

data/.travis.yml

@@ -24,3 +24,5 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
+    - rvm: jruby-19mode
+      os: osx

data/ChangeLog

@@ -1,3 +1,11 @@
+2015-10-24 version 0.7.0:
+
+* Add extention types support.
+* Fix to share almost all test cases between CRuby and JRuby implementations.
+* Fixed JRuby implementation to raise UnknownExtTypeError for unregistered ext type ids
+  instead to generate MessagePack::ExtensionValue instances.
+  (Specify `allow_unknown_ext: true` as unpacker option for v0.6.x behavior.)
+
 2015-07-22 version 0.6.2:
 
 * Fix release workflow: Ruby 2.1 and 2.2 are supported for Windows (2.0 is omitted)

data/README.rdoc

@@ -36,7 +36,6 @@ or build msgpack-ruby and install:
 * Exchange objects between software components written in different languages
   * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
 
-
 = Portability
 
 MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
@@ -92,6 +91,32 @@ or event-driven style which works well with EventMachine:
 
 See {API reference}[http://ruby.msgpack.org/MessagePack/Unpacker.html] for details.
 
+= Extension Types
+
+Packer and Unpacker support {Extension types of MessagePack}[https://github.com/msgpack/msgpack/blob/master/spec.md#types-extension-type].
+
+    # register how to serialize custom class at first
+    pk = MessagePack::Packer.new(io)
+    pk.register_type(0x01, MyClass1, :to_msgpack_ext) # equal to pk.register_type(0x01, MyClass)
+    pk.register_type(0x02, MyClass2){|obj| obj.how_to_serialize() } # blocks also available
+    
+    # almost same API for unpacker
+    uk = MessagePack::Unpacker.new()
+    uk.register_type(0x01, MyClass1, :from_msgpack_ext)
+    uk.register_type(0x02){|data| MyClass2.create_from_serialized_data(data) }
+
+MessagePack::Factory is to create packer and unpacker which have same extention types.
+
+    factory = MessagePack::Factory.new
+    factory.register_type(0x01, MyClass1) # same with next line
+    factory.register_type(0x01, MyClass1, packer: :to_msgpack_ext, unpacker: :from_msgpack_ext)
+    pk = factory.packer(options_for_packer)
+    uk = factory.unpacker(options_for_unpacker)
+
+For *MessagePack.pack* and *MessagePack.unpack*, default packer/unpacker refer *MessagePack::DefaultFactory*. Call *MessagePack::DefaultFactory.register_type* to enable types process globally.
+
+    MessagePack::DefaultFactory.register_type(0x03, MyClass3)
+    MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
 
 = Buffer API
 
@@ -129,9 +154,18 @@ MessagePack mingw32/64 rubygems build process uses {rake-compiler-dock}[https://
 
 Once this step successes, target gems exist in pkg/msgpack-*-{x86,x64}-mingw32.gem.
 
+== Updating documents
+
+Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
+Following commands update documents in gh-pages branch:
+
+    bundle exec rake doc
+    git checkout gh-pages
+    cp doc/* ./ -a
+
 = Copyright
 
 Author::    Sadayuki Furuhashi <frsyuki@gmail.com>
-Copyright:: Copyright (c) 2008-2013 Sadayuki Furuhashi
+Copyright:: Copyright (c) 2008-2015 Sadayuki Furuhashi
 License::   Apache License, Version 2.0
 

data/ext/java/org/msgpack/jruby/Decoder.java

@@ -12,6 +12,7 @@ import org.jruby.RubyObject;
 import org.jruby.RubyClass;
 import org.jruby.RubyBignum;
 import org.jruby.RubyString;
+import org.jruby.RubyArray;
 import org.jruby.RubyHash;
 import org.jruby.exceptions.RaiseException;
 import org.jruby.runtime.builtin.IRubyObject;
@@ -29,33 +30,58 @@ public class Decoder implements Iterator<IRubyObject> {
   private final Encoding utf8Encoding;
   private final RubyClass unpackErrorClass;
   private final RubyClass underflowErrorClass;
+  private final RubyClass malformedFormatErrorClass;
+  private final RubyClass stackErrorClass;
   private final RubyClass unexpectedTypeErrorClass;
+  private final RubyClass unknownExtTypeErrorClass;
 
+  private ExtensionRegistry registry;
   private ByteBuffer buffer;
   private boolean symbolizeKeys;
+  private boolean allowUnknownExt;
 
   public Decoder(Ruby runtime) {
-    this(runtime, new byte[] {}, 0, 0);
+    this(runtime, null, new byte[] {}, 0, 0, false, false);
+  }
+
+  public Decoder(Ruby runtime, ExtensionRegistry registry) {
+    this(runtime, registry, new byte[] {}, 0, 0, false, false);
   }
 
   public Decoder(Ruby runtime, byte[] bytes) {
-    this(runtime, bytes, 0, bytes.length);
+    this(runtime, null, bytes, 0, bytes.length, false, false);
+  }
+
+  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes) {
+    this(runtime, registry, bytes, 0, bytes.length, false, false);
+  }
+
+  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean allowUnknownExt) {
+    this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, allowUnknownExt);
+  }
+
+  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length) {
+    this(runtime, registry, bytes, offset, length, false, false);
   }
 
-  public Decoder(Ruby runtime, byte[] bytes, int offset, int length) {
+  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean allowUnknownExt) {
     this.runtime = runtime;
+    this.registry = registry;
+    this.symbolizeKeys = symbolizeKeys;
+    this.allowUnknownExt = allowUnknownExt;
     this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding();
     this.utf8Encoding = UTF8Encoding.INSTANCE;
     this.unpackErrorClass = runtime.getModule("MessagePack").getClass("UnpackError");
     this.underflowErrorClass = runtime.getModule("MessagePack").getClass("UnderflowError");
+    this.malformedFormatErrorClass = runtime.getModule("MessagePack").getClass("MalformedFormatError");
+    this.stackErrorClass = runtime.getModule("MessagePack").getClass("StackError");
     this.unexpectedTypeErrorClass = runtime.getModule("MessagePack").getClass("UnexpectedTypeError");
+    this.unknownExtTypeErrorClass = runtime.getModule("MessagePack").getClass("UnknownExtTypeError");
+    this.symbolizeKeys = symbolizeKeys;
+    this.allowUnknownExt = allowUnknownExt;
     feed(bytes, offset, length);
   }
 
-  public void symbolizeKeys(boolean symbolize) {
-    this.symbolizeKeys = symbolize;
-  }
-
   public void feed(byte[] bytes) {
     feed(bytes, 0, bytes.length);
   }
@@ -73,7 +99,7 @@ public class Decoder implements Iterator<IRubyObject> {
   }
 
   public void reset() {
-    buffer.rewind();
+    buffer = null;
   }
 
   public int offset() {
@@ -118,7 +144,20 @@ public class Decoder implements Iterator<IRubyObject> {
   private IRubyObject consumeExtension(int size) {
     int type = buffer.get();
     byte[] payload = readBytes(size);
-    return ExtensionValue.newExtensionValue(runtime, type, payload);
+
+    if (registry != null) {
+      IRubyObject proc = registry.lookupUnpackerByTypeId(type);
+      if (proc != null) {
+        ByteList byteList = new ByteList(payload, runtime.getEncodingService().getAscii8bitEncoding());
+        return proc.callMethod(runtime.getCurrentContext(), "call", runtime.newString(byteList));
+      }
+    }
+
+    if (this.allowUnknownExt) {
+      return ExtensionValue.newExtensionValue(runtime, type, payload);
+    }
+
+    throw runtime.newRaiseException(unknownExtTypeErrorClass, "unexpected extension type");
   }
 
   private byte[] readBytes(int size) {
@@ -142,11 +181,11 @@ public class Decoder implements Iterator<IRubyObject> {
     try {
       byte b = buffer.get();
       if ((b & 0xf0) == 0x90) {
-	return runtime.newFixnum(b & 0x0f);
+        return runtime.newFixnum(b & 0x0f);
       } else if (b == ARY16) {
-	return runtime.newFixnum(buffer.getShort() & 0xffff);
+        return runtime.newFixnum(buffer.getShort() & 0xffff);
       } else if (b == ARY32) {
-	return runtime.newFixnum(buffer.getInt());
+        return runtime.newFixnum(buffer.getInt());
       }
       throw runtime.newRaiseException(unexpectedTypeErrorClass, "unexpected type");
     } catch (RaiseException re) {
@@ -163,11 +202,11 @@ public class Decoder implements Iterator<IRubyObject> {
     try {
       byte b = buffer.get();
       if ((b & 0xf0) == 0x80) {
-	return runtime.newFixnum(b & 0x0f);
+        return runtime.newFixnum(b & 0x0f);
       } else if (b == MAP16) {
-	return runtime.newFixnum(buffer.getShort() & 0xffff);
+        return runtime.newFixnum(buffer.getShort() & 0xffff);
       } else if (b == MAP32) {
-	return runtime.newFixnum(buffer.getInt());
+        return runtime.newFixnum(buffer.getInt());
       }
       throw runtime.newRaiseException(unexpectedTypeErrorClass, "unexpected type");
     } catch (RaiseException re) {
@@ -233,7 +272,7 @@ public class Decoder implements Iterator<IRubyObject> {
       default: return runtime.newFixnum(b);
       }
       buffer.position(position);
-      throw runtime.newRaiseException(unpackErrorClass, "Illegal byte sequence");
+      throw runtime.newRaiseException(malformedFormatErrorClass, "Illegal byte sequence");
     } catch (RaiseException re) {
       buffer.position(position);
       throw re;

data/ext/java/org/msgpack/jruby/Encoder.java

@@ -34,15 +34,21 @@ public class Encoder {
   private final Encoding binaryEncoding;
   private final Encoding utf8Encoding;
   private final boolean compatibilityMode;
+  private final ExtensionRegistry registry;
 
   private ByteBuffer buffer;
 
-  public Encoder(Ruby runtime, boolean compatibilityMode) {
+  public Encoder(Ruby runtime, boolean compatibilityMode, ExtensionRegistry registry) {
     this.runtime = runtime;
     this.buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE);
     this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding();
     this.utf8Encoding = UTF8Encoding.INSTANCE;
     this.compatibilityMode = compatibilityMode;
+    this.registry = registry;
+  }
+
+  public boolean isCompatibilityMode() {
+    return compatibilityMode;
   }
 
   private void ensureRemainingCapacity(int c) {
@@ -107,7 +113,7 @@ public class Encoder {
     } else if (object instanceof ExtensionValue) {
       appendExtensionValue((ExtensionValue) object);
     } else {
-      appendCustom(object, destination);
+      appendOther(object, destination);
     }
   }
 
@@ -295,12 +301,7 @@ public class Encoder {
     }
   }
 
-  private void appendExtensionValue(ExtensionValue object) {
-    long type = ((RubyFixnum)object.get_type()).getLongValue();
-    if (type < -128 || type > 127) {
-	    throw object.getRuntime().newRangeError(String.format("integer %d too big to convert to `signed char'", type));
-    }
-    ByteList payloadBytes = ((RubyString)object.payload()).getByteList();
+  private void appendExt(int type, ByteList payloadBytes) {
     int payloadSize = payloadBytes.length();
     int outputSize = 0;
     boolean fixSize = payloadSize == 1 || payloadSize == 2 || payloadSize == 4 || payloadSize == 8 || payloadSize == 16;
@@ -338,6 +339,28 @@ public class Encoder {
     buffer.put(payloadBytes.unsafeBytes(), payloadBytes.begin(), payloadSize);
   }
 
+  private void appendExtensionValue(ExtensionValue object) {
+    long type = ((RubyFixnum)object.get_type()).getLongValue();
+    if (type < -128 || type > 127) {
+      throw object.getRuntime().newRangeError(String.format("integer %d too big to convert to `signed char'", type));
+    }
+    ByteList payloadBytes = ((RubyString)object.payload()).getByteList();
+    appendExt((int) type, payloadBytes);
+  }
+
+  private void appendOther(IRubyObject object, IRubyObject destination) {
+    if (registry != null) {
+      IRubyObject[] pair = registry.lookupPackerByClass(object.getType());
+      if (pair != null) {
+        RubyString bytes = pair[0].callMethod(runtime.getCurrentContext(), "call", object).asString();
+        int type = (int) ((RubyFixnum) pair[1]).getLongValue();
+        appendExt(type, bytes.getByteList());
+        return;
+      }
+    }
+    appendCustom(object, destination);
+  }
+
   private void appendCustom(IRubyObject object, IRubyObject destination) {
     if (destination == null) {
       IRubyObject result = object.callMethod(runtime.getCurrentContext(), "to_msgpack");

data/ext/java/org/msgpack/jruby/ExtensionRegistry.java

@@ -0,0 +1,159 @@
+package org.msgpack.jruby;
+
+import org.jruby.Ruby;
+import org.jruby.RubyHash;
+import org.jruby.RubyArray;
+import org.jruby.RubyClass;
+import org.jruby.RubyFixnum;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+import java.util.Map;
+import java.util.HashMap;
+
+public class ExtensionRegistry {
+  private final Map<RubyClass, ExtensionEntry> extensionsByClass;
+  private final Map<RubyClass, ExtensionEntry> extensionsByAncestor;
+  private final ExtensionEntry[] extensionsByTypeId;
+
+  public ExtensionRegistry() {
+    this(new HashMap<RubyClass, ExtensionEntry>());
+  }
+
+  private ExtensionRegistry(Map<RubyClass, ExtensionEntry> extensionsByClass) {
+    this.extensionsByClass = new HashMap<RubyClass, ExtensionEntry>(extensionsByClass);
+    this.extensionsByAncestor = new HashMap<RubyClass, ExtensionEntry>();
+    this.extensionsByTypeId = new ExtensionEntry[256];
+    for (ExtensionEntry entry : extensionsByClass.values()) {
+      if (entry.hasUnpacker()) {
+        extensionsByTypeId[entry.getTypeId() + 128] = entry;
+      }
+    }
+  }
+
+  public ExtensionRegistry dup() {
+    return new ExtensionRegistry(extensionsByClass);
+  }
+
+  public IRubyObject toInternalPackerRegistry(ThreadContext ctx) {
+    RubyHash hash = RubyHash.newHash(ctx.getRuntime());
+    for (RubyClass extensionClass : extensionsByClass.keySet()) {
+      ExtensionEntry entry = extensionsByClass.get(extensionClass);
+      if (entry.hasPacker()) {
+        hash.put(extensionClass, entry.toPackerTuple(ctx));
+      }
+    }
+    return hash;
+  }
+
+  public IRubyObject toInternalUnpackerRegistry(ThreadContext ctx) {
+    RubyHash hash = RubyHash.newHash(ctx.getRuntime());
+    for (int typeIdIndex = 0 ; typeIdIndex < 256 ; typeIdIndex++) {
+      ExtensionEntry entry = extensionsByTypeId[typeIdIndex];
+      if (entry != null && entry.hasUnpacker()) {
+        IRubyObject typeId = RubyFixnum.newFixnum(ctx.getRuntime(), typeIdIndex - 128);
+        hash.put(typeId, entry.toUnpackerTuple(ctx));
+      }
+    }
+    return hash;
+  }
+
+  public void put(RubyClass cls, int typeId, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
+    ExtensionEntry entry = new ExtensionEntry(cls, typeId, packerProc, packerArg, unpackerProc, unpackerArg);
+    extensionsByClass.put(cls, entry);
+    extensionsByTypeId[typeId + 128] = entry;
+    extensionsByAncestor.clear();
+  }
+
+  public IRubyObject lookupUnpackerByTypeId(int typeId) {
+    ExtensionEntry e = extensionsByTypeId[typeId + 128];
+    if (e != null && e.hasUnpacker()) {
+      return e.getUnpackerProc();
+    } else {
+      return null;
+    }
+  }
+
+  public IRubyObject[] lookupPackerByClass(RubyClass cls) {
+    ExtensionEntry e = extensionsByClass.get(cls);
+    if (e == null) {
+      e = extensionsByAncestor.get(cls);
+    }
+    if (e == null) {
+      e = findEntryByClassOrAncestor(cls);
+      if (e != null) {
+        extensionsByAncestor.put(e.getExtensionClass(), e);
+      }
+    }
+    if (e != null && e.hasPacker()) {
+      return e.toPackerProcTypeIdPair(cls.getRuntime().getCurrentContext());
+    } else {
+      return null;
+    }
+  }
+
+  private ExtensionEntry findEntryByClassOrAncestor(final RubyClass cls) {
+    ThreadContext ctx = cls.getRuntime().getCurrentContext();
+    for (RubyClass extensionClass : extensionsByClass.keySet()) {
+      RubyArray ancestors = (RubyArray) cls.callMethod(ctx, "ancestors");
+      if (ancestors.callMethod(ctx, "include?", extensionClass).isTrue()) {
+        return extensionsByClass.get(extensionClass);
+      }
+    }
+    return null;
+  }
+
+  private static class ExtensionEntry {
+    private final RubyClass cls;
+    private final int typeId;
+    private final IRubyObject packerProc;
+    private final IRubyObject packerArg;
+    private final IRubyObject unpackerProc;
+    private final IRubyObject unpackerArg;
+
+    public ExtensionEntry(RubyClass cls, int typeId, IRubyObject packerProc, IRubyObject packerArg, IRubyObject unpackerProc, IRubyObject unpackerArg) {
+      this.cls = cls;
+      this.typeId = typeId;
+      this.packerProc = packerProc;
+      this.packerArg = packerArg;
+      this.unpackerProc = unpackerProc;
+      this.unpackerArg = unpackerArg;
+    }
+
+    public RubyClass getExtensionClass() {
+      return cls;
+    }
+
+    public int getTypeId() {
+      return typeId;
+    }
+
+    public boolean hasPacker() {
+      return packerProc != null;
+    }
+
+    public boolean hasUnpacker() {
+      return unpackerProc != null;
+    }
+
+    public IRubyObject getPackerProc() {
+      return packerProc;
+    }
+
+    public IRubyObject getUnpackerProc() {
+      return unpackerProc;
+    }
+
+    public RubyArray toPackerTuple(ThreadContext ctx) {
+      return RubyArray.newArray(ctx.getRuntime(), new IRubyObject[] {RubyFixnum.newFixnum(ctx.getRuntime(), typeId), packerProc, packerArg});
+    }
+
+    public RubyArray toUnpackerTuple(ThreadContext ctx) {
+      return RubyArray.newArray(ctx.getRuntime(), new IRubyObject[] {cls, unpackerProc, unpackerArg});
+    }
+
+    public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) {
+      return new IRubyObject[] {packerProc, RubyFixnum.newFixnum(ctx.getRuntime(), typeId)};
+    }
+  }
+}