ed-precompiled_msgpack 1.8.0

data/ext/java/org/msgpack/jruby/Encoder.java

@@ -0,0 +1,456 @@
+package org.msgpack.jruby;
+
+
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.RubyModule;
+import org.jruby.RubyNil;
+import org.jruby.RubyBoolean;
+import org.jruby.RubyNumeric;
+import org.jruby.RubyBignum;
+import org.jruby.RubyInteger;
+import org.jruby.RubyFixnum;
+import org.jruby.RubyFloat;
+import org.jruby.RubyString;
+import org.jruby.RubySymbol;
+import org.jruby.RubyArray;
+import org.jruby.RubyHash;
+import org.jruby.RubyEncoding;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.util.ByteList;
+
+import org.jcodings.Encoding;
+import org.jcodings.specific.UTF8Encoding;
+
+import static org.msgpack.jruby.Types.*;
+
+public class Encoder {
+
+  private static final int CACHE_LINE_SIZE = 64;
+  private static final int ARRAY_HEADER_SIZE = 24;
+
+  private final Ruby runtime;
+  private final Encoding binaryEncoding;
+  private final Encoding utf8Encoding;
+  private final boolean compatibilityMode;
+  private final ExtensionRegistry registry;
+  private final Packer packer;
+
+  public boolean hasSymbolExtType;
+  private boolean hasBigintExtType;
+  private boolean recursiveExtension;
+
+  private ByteBuffer buffer;
+
+  public Encoder(Ruby runtime, Packer packer, boolean compatibilityMode, ExtensionRegistry registry, boolean hasSymbolExtType, boolean hasBigintExtType) {
+    this.packer = packer;
+    this.runtime = runtime;
+    this.buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE);
+    this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding();
+    this.utf8Encoding = UTF8Encoding.INSTANCE;
+    this.compatibilityMode = compatibilityMode;
+    this.registry = registry;
+    this.hasSymbolExtType = hasSymbolExtType;
+    this.hasBigintExtType = hasBigintExtType;
+  }
+
+  public boolean isCompatibilityMode() {
+    return compatibilityMode;
+  }
+
+  private void ensureRemainingCapacity(int c) {
+    if (buffer.remaining() < c) {
+      int newLength = Math.max(buffer.capacity() + (buffer.capacity() >> 1), buffer.capacity() + c);
+      newLength += CACHE_LINE_SIZE - ((ARRAY_HEADER_SIZE + newLength) % CACHE_LINE_SIZE);
+      buffer = ByteBuffer.allocate(newLength).put(buffer.array(), 0, buffer.position());
+    }
+  }
+
+  private IRubyObject readRubyString() {
+    if (recursiveExtension) {
+      // If recursiveExtension is true, it means we re-entered encode, so we MUST NOT flush the buffer.
+	  // Instead we return an empty string to act as a null object for the caller. The buffer will actually
+	  // be flushed once we're done serializing the recursive extension.
+	  // All other method that consume the buffer should do so through readRubyString or implement the same logic.
+      return runtime.newString();
+    } else {
+      IRubyObject str = runtime.newString(new ByteList(buffer.array(), 0, buffer.position(), binaryEncoding, false));
+      buffer.clear();
+      return str;
+    }
+  }
+
+  public IRubyObject encode(IRubyObject object) {
+    appendObject(object);
+    return readRubyString();
+  }
+
+  public IRubyObject encode(IRubyObject object, IRubyObject destination) {
+    appendObject(object, destination);
+    return readRubyString();
+  }
+
+  public IRubyObject encodeArrayHeader(int size) {
+    appendArrayHeader(size);
+    return readRubyString();
+  }
+
+  public IRubyObject encodeMapHeader(int size) {
+    appendHashHeader(size);
+    return readRubyString();
+  }
+
+  public IRubyObject encodeBinHeader(int size) {
+    appendStringHeader(size, true);
+    return readRubyString();
+  }
+
+  public IRubyObject encodeFloat32(RubyNumeric numeric) {
+    appendFloat32(numeric);
+    return readRubyString();
+  }
+
+  private void appendObject(IRubyObject object) {
+    appendObject(object, null);
+  }
+
+  private void appendObject(IRubyObject object, IRubyObject destination) {
+    if (object == null || object instanceof RubyNil) {
+      ensureRemainingCapacity(1);
+      buffer.put(NIL);
+    } else if (object instanceof RubyBoolean) {
+      ensureRemainingCapacity(1);
+      buffer.put(((RubyBoolean) object).isTrue() ? TRUE : FALSE);
+    } else if (object instanceof RubyBignum) {
+      appendBignum((RubyBignum) object);
+    } else if (object instanceof RubyInteger) {
+      appendInteger((RubyInteger) object);
+    } else if (object instanceof RubyFloat) {
+      appendFloat((RubyFloat) object);
+    } else if (object instanceof RubyString) {
+      if (object.getType() == runtime.getString() || !tryAppendWithExtTypeLookup(object)) {
+        appendString((RubyString) object);
+      }
+    } else if (object instanceof RubySymbol) {
+      if (hasSymbolExtType) {
+        appendOther(object, destination);
+      } else {
+        appendString(((RubySymbol) object).asString());
+      }
+    } else if (object instanceof RubyArray) {
+      if (object.getType() == runtime.getArray() || !tryAppendWithExtTypeLookup(object)) {
+        appendArray((RubyArray) object);
+      }
+    } else if (object instanceof RubyHash) {
+      if (object.getType() == runtime.getHash() || !tryAppendWithExtTypeLookup(object)) {
+        appendHash((RubyHash) object);
+      }
+    } else if (object instanceof ExtensionValue) {
+      appendExtensionValue((ExtensionValue) object);
+    } else {
+      appendOther(object, destination);
+    }
+  }
+
+  private void appendBignum(RubyBignum object) {
+    BigInteger value = object.getBigIntegerValue();
+    if (value.compareTo(RubyBignum.LONG_MIN) < 0 || value.compareTo(RubyBignum.LONG_MAX) > 0) {
+      if (value.bitLength() > 64 || (value.bitLength() > 63 && value.signum() < 0)) {
+        if (hasBigintExtType && tryAppendWithExtTypeLookup(object)) {
+          return;
+        }
+        throw runtime.newRangeError(String.format("Cannot pack big integer: %s", value));
+      }
+      ensureRemainingCapacity(9);
+      buffer.put(value.signum() < 0 ? INT64 : UINT64);
+      byte[] b = value.toByteArray();
+      buffer.put(b, b.length - 8, 8);
+    } else {
+      appendInteger(object);
+    }
+  }
+
+  private void appendInteger(RubyInteger object) {
+    long value = object.getLongValue();
+    if (value < 0) {
+      if (value < Short.MIN_VALUE) {
+        if (value < Integer.MIN_VALUE) {
+          ensureRemainingCapacity(9);
+          buffer.put(INT64);
+          buffer.putLong(value);
+        } else {
+          ensureRemainingCapacity(5);
+          buffer.put(INT32);
+          buffer.putInt((int) value);
+        }
+      } else if (value >= -0x20L) {
+        ensureRemainingCapacity(1);
+        byte b = (byte) (value | 0xe0);
+        buffer.put(b);
+      } else if (value < Byte.MIN_VALUE) {
+        ensureRemainingCapacity(3);
+        buffer.put(INT16);
+        buffer.putShort((short) value);
+      } else {
+        ensureRemainingCapacity(2);
+        buffer.put(INT8);
+        buffer.put((byte) value);
+      }
+    } else {
+      if (value < 0x10000L) {
+        if (value < 128L) {
+          ensureRemainingCapacity(1);
+          buffer.put((byte) value);
+        } else if (value < 0x100L) {
+          ensureRemainingCapacity(2);
+          buffer.put(UINT8);
+          buffer.put((byte) value);
+        } else {
+          ensureRemainingCapacity(3);
+          buffer.put(UINT16);
+          buffer.putShort((short) value);
+        }
+      } else if (value < 0x100000000L) {
+        ensureRemainingCapacity(5);
+        buffer.put(UINT32);
+        buffer.putInt((int) value);
+      } else {
+        ensureRemainingCapacity(9);
+        buffer.put(INT64);
+        buffer.putLong(value);
+      }
+    }
+  }
+
+  private void appendFloat(RubyFloat object) {
+    double value = object.getDoubleValue();
+    //TODO: msgpack-ruby original does encode this value as Double, not float
+    // float f = (float) value;
+    // if (Double.compare(f, value) == 0) {
+    //   ensureRemainingCapacity(5);
+    //   buffer.put(FLOAT32);
+    //   buffer.putFloat(f);
+    // } else {
+      ensureRemainingCapacity(9);
+      buffer.put(FLOAT64);
+      buffer.putDouble(value);
+    // }
+  }
+
+  private void appendFloat32(RubyNumeric object) {
+    float value = (float) object.getDoubleValue();
+    ensureRemainingCapacity(5);
+    buffer.put(FLOAT32);
+    buffer.putFloat(value);
+  }
+
+  private void appendStringHeader(int length, boolean binary) {
+    if (length < 32 && !binary) {
+      ensureRemainingCapacity(1 + length);
+      buffer.put((byte) (length | FIXSTR));
+    } else if (length <= 0xff && !compatibilityMode) {
+      ensureRemainingCapacity(2 + length);
+      buffer.put(binary ? BIN8 : STR8);
+      buffer.put((byte) length);
+    } else if (length <= 0xffff) {
+      ensureRemainingCapacity(3 + length);
+      buffer.put(binary ? BIN16 : STR16);
+      buffer.putShort((short) length);
+    } else {
+      ensureRemainingCapacity(5 + length);
+      buffer.put(binary ? BIN32 : STR32);
+      buffer.putInt(length);
+    }
+  }
+
+  private void appendString(RubyString object) {
+    Encoding encoding = object.getEncoding();
+    boolean binary = !compatibilityMode && encoding == binaryEncoding;
+    if (encoding != utf8Encoding && encoding != binaryEncoding) {
+      object = (RubyString)(object).encode(runtime.getCurrentContext(), runtime.getEncodingService().getEncoding(utf8Encoding));
+    }
+    ByteList bytes = object.getByteList();
+    int length = bytes.length();
+    appendStringHeader(length, binary);
+    buffer.put(bytes.unsafeBytes(), bytes.begin(), length);
+  }
+
+  private void appendArray(RubyArray<?> object) {
+    appendArrayHeader(object);
+    appendArrayElements(object);
+  }
+
+  private void appendArrayHeader(RubyArray<?> object) {
+    appendArrayHeader(object.size());
+  }
+
+  private void appendArrayHeader(int size) {
+    if (size < 16) {
+      ensureRemainingCapacity(1);
+      buffer.put((byte) (size | 0x90));
+    } else if (size < 0x10000) {
+      ensureRemainingCapacity(3);
+      buffer.put(ARY16);
+      buffer.putShort((short) size);
+    } else {
+      ensureRemainingCapacity(5);
+      buffer.put(ARY32);
+      buffer.putInt(size);
+    }
+  }
+
+  private void appendArrayElements(RubyArray<?> object) {
+    int size = object.size();
+    for (int i = 0; i < size; i++) {
+      appendObject(object.eltOk(i));
+    }
+  }
+
+  private void appendHash(RubyHash object) {
+    appendHashHeader(object);
+    appendHashElements(object);
+  }
+
+  private void appendHashHeader(RubyHash object) {
+    appendHashHeader(object.size());
+  }
+
+  private void appendHashHeader(int size) {
+    if (size < 16) {
+      ensureRemainingCapacity(1);
+      buffer.put((byte) (size | 0x80));
+    } else if (size < 0x10000) {
+      ensureRemainingCapacity(3);
+      buffer.put(MAP16);
+      buffer.putShort((short) size);
+    } else {
+      ensureRemainingCapacity(5);
+      buffer.put(MAP32);
+      buffer.putInt(size);
+    }
+  }
+
+  private void appendHashElements(RubyHash object) {
+    int size = object.size();
+    HashVisitor visitor = new HashVisitor(size);
+    object.visitAll(visitor);
+    if (visitor.remain != 0) {
+      object.getRuntime().newConcurrencyError("Hash size changed while packing");
+    }
+  }
+
+  private class HashVisitor extends RubyHash.Visitor {
+    public int remain;
+
+    public HashVisitor(int size) {
+      remain = size;
+    }
+
+    public void visit(IRubyObject key, IRubyObject value) {
+      if (remain-- > 0) {
+        appendObject(key);
+        appendObject(value);
+      }
+    }
+  }
+
+  private void appendExt(int type, ByteList payloadBytes) {
+    int payloadSize = payloadBytes.length();
+    int outputSize = 0;
+    boolean fixSize = payloadSize == 1 || payloadSize == 2 || payloadSize == 4 || payloadSize == 8 || payloadSize == 16;
+    if (fixSize) {
+      outputSize = 2 + payloadSize;
+    } else if (payloadSize < 0x100) {
+      outputSize = 3 + payloadSize;
+    } else if (payloadSize < 0x10000) {
+      outputSize = 4 + payloadSize;
+    } else {
+      outputSize = 6 + payloadSize;
+    }
+    ensureRemainingCapacity(outputSize);
+    if (payloadSize == 1) {
+      buffer.put(FIXEXT1);
+    } else if (payloadSize == 2) {
+      buffer.put(FIXEXT2);
+    } else if (payloadSize == 4) {
+      buffer.put(FIXEXT4);
+    } else if (payloadSize == 8) {
+      buffer.put(FIXEXT8);
+    } else if (payloadSize == 16) {
+      buffer.put(FIXEXT16);
+    } else if (payloadSize < 0x100) {
+      buffer.put(VAREXT8);
+      buffer.put((byte) payloadSize);
+    } else if (payloadSize < 0x10000) {
+      buffer.put(VAREXT16);
+      buffer.putShort((short) payloadSize);
+    } else {
+      buffer.put(VAREXT32);
+      buffer.putInt(payloadSize);
+    }
+    buffer.put((byte) type);
+    buffer.put(payloadBytes.unsafeBytes(), payloadBytes.begin(), payloadSize);
+  }
+
+  private void appendExtensionValue(ExtensionValue object) {
+    long type = ((RubyFixnum)object.get_type()).getLongValue();
+    if (type < -128 || type > 127) {
+      throw object.getRuntime().newRangeError(String.format("integer %d too big to convert to `signed char'", type));
+    }
+    ByteList payloadBytes = ((RubyString)object.payload()).getByteList();
+    appendExt((int) type, payloadBytes);
+  }
+
+  private boolean tryAppendWithExtTypeLookup(IRubyObject object) {
+    if (registry != null) {
+      ExtensionRegistry.ExtensionEntry entry = registry.lookupExtensionForObject(object);
+      if (entry != null) {
+        IRubyObject proc = entry.getPackerProc();
+        int type = entry.getTypeId();
+
+        if (entry.isRecursive()) {
+          ByteBuffer oldBuffer = buffer;
+          buffer = ByteBuffer.allocate(CACHE_LINE_SIZE - ARRAY_HEADER_SIZE);
+          boolean previousRecursiveExtension = recursiveExtension;
+          recursiveExtension = true;
+
+          ByteList payload;
+          try {
+            IRubyObject args[] = { object, packer };
+            proc.callMethod(runtime.getCurrentContext(), "call", args);
+            payload = new ByteList(buffer.array(), 0, buffer.position(), binaryEncoding, false);
+          } finally {
+            recursiveExtension = previousRecursiveExtension;
+            buffer = oldBuffer;
+          }
+          appendExt(type, payload);
+        } else {
+          RubyString bytes = proc.callMethod(runtime.getCurrentContext(), "call", object).asString();
+          appendExt(type, bytes.getByteList());
+        }
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private void appendOther(IRubyObject object, IRubyObject destination) {
+    if (!tryAppendWithExtTypeLookup(object)) { appendCustom(object, destination); }
+  }
+
+  private void appendCustom(IRubyObject object, IRubyObject destination) {
+    if (destination == null) {
+      IRubyObject result = object.callMethod(runtime.getCurrentContext(), "to_msgpack");
+      ByteList bytes = result.asString().getByteList();
+      int length = bytes.length();
+      ensureRemainingCapacity(length);
+      buffer.put(bytes.unsafeBytes(), bytes.begin(), length);
+    } else {
+      object.callMethod(runtime.getCurrentContext(), "to_msgpack", destination);
+    }
+  }
+}

data/ext/java/org/msgpack/jruby/ExtensionRegistry.java

@@ -0,0 +1,167 @@
+package org.msgpack.jruby;
+
+import org.jruby.Ruby;
+import org.jruby.RubyHash;
+import org.jruby.RubyArray;
+import org.jruby.RubyModule;
+import org.jruby.RubyFixnum;
+import org.jruby.RubySymbol;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+
+import java.util.Map;
+import java.util.HashMap;
+
+public class ExtensionRegistry {
+  private final Map<RubyModule, ExtensionEntry> extensionsByModule;
+  private final Map<RubyModule, ExtensionEntry> extensionsByAncestor;
+  private final ExtensionEntry[] extensionsByTypeId;
+
+  public ExtensionRegistry() {
+    this(new HashMap<RubyModule, ExtensionEntry>(), new ExtensionEntry[256]);
+  }
+
+  private ExtensionRegistry(Map<RubyModule, ExtensionEntry> extensionsByModule, ExtensionEntry[] extensionsByTypeId) {
+    this.extensionsByModule = new HashMap<RubyModule, ExtensionEntry>(extensionsByModule);
+    this.extensionsByAncestor = new HashMap<RubyModule, ExtensionEntry>();
+    this.extensionsByTypeId = extensionsByTypeId.clone();
+  }
+
+  public ExtensionRegistry dup() {
+    return new ExtensionRegistry(extensionsByModule, extensionsByTypeId);
+  }
+
+  public IRubyObject toInternalPackerRegistry(ThreadContext ctx) {
+    RubyHash hash = RubyHash.newHash(ctx.runtime);
+    for (RubyModule extensionModule : extensionsByModule.keySet()) {
+      ExtensionEntry entry = extensionsByModule.get(extensionModule);
+      if (entry.hasPacker()) {
+        hash.put(extensionModule, entry.toPackerTuple(ctx));
+      }
+    }
+    return hash;
+  }
+
+  public IRubyObject toInternalUnpackerRegistry(ThreadContext ctx) {
+    RubyHash hash = RubyHash.newHash(ctx.runtime);
+    for (int typeIdIndex = 0 ; typeIdIndex < 256 ; typeIdIndex++) {
+      ExtensionEntry entry = extensionsByTypeId[typeIdIndex];
+      if (entry != null && entry.hasUnpacker()) {
+        IRubyObject typeId = RubyFixnum.newFixnum(ctx.runtime, typeIdIndex - 128);
+        hash.put(typeId, entry.toUnpackerTuple(ctx));
+      }
+    }
+    return hash;
+  }
+
+  public void put(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
+    ExtensionEntry entry = new ExtensionEntry(mod, typeId, recursive, packerProc, unpackerProc);
+    extensionsByModule.put(mod, entry);
+    extensionsByTypeId[typeId + 128] = entry;
+    extensionsByAncestor.clear();
+  }
+
+  public ExtensionEntry lookupExtensionByTypeId(int typeId) {
+    ExtensionEntry e = extensionsByTypeId[typeId + 128];
+    if (e != null && e.hasUnpacker()) {
+      return e;
+    }
+    return null;
+  }
+
+  public ExtensionEntry lookupExtensionForObject(IRubyObject object) {
+    RubyModule lookupClass = null;
+    ExtensionEntry entry = null;
+    /*
+     * Objects of type Integer (Fixnum, Bignum), Float, Symbol and frozen
+     * String have no singleton class and raise a TypeError when trying to get
+     * it.
+     */
+    lookupClass = object.getMetaClass();
+    entry = extensionsByModule.get(lookupClass);
+    if (entry != null && entry.hasPacker()) {
+      return entry;
+    }
+
+    RubyModule realClass = object.getType();
+    if (realClass != lookupClass) {
+      entry = extensionsByModule.get(realClass);
+      if (entry != null && entry.hasPacker()) {
+        return entry;
+      }
+    }
+
+    entry = findEntryByModuleOrAncestor(lookupClass);
+    if (entry != null && entry.hasPacker()) {
+      return entry;
+    }
+    return null;
+  }
+
+  private ExtensionEntry findEntryByModuleOrAncestor(final RubyModule mod) {
+    ThreadContext ctx = mod.getRuntime().getCurrentContext();
+    for (RubyModule extensionModule : extensionsByModule.keySet()) {
+      RubyArray<?> ancestors = (RubyArray)mod.callMethod(ctx, "ancestors");
+      if (ancestors.callMethod(ctx, "include?", extensionModule).isTrue()) {
+        return extensionsByModule.get(extensionModule);
+      }
+    }
+    return null;
+  }
+
+  public static class ExtensionEntry {
+    private final RubyModule mod;
+    private final int typeId;
+    private final boolean recursive;
+    private final IRubyObject packerProc;
+    private final IRubyObject unpackerProc;
+
+    public ExtensionEntry(RubyModule mod, int typeId, boolean recursive, IRubyObject packerProc, IRubyObject unpackerProc) {
+      this.mod = mod;
+      this.typeId = typeId;
+      this.recursive = recursive;
+      this.packerProc = packerProc;
+      this.unpackerProc = unpackerProc;
+    }
+
+    public RubyModule getExtensionModule() {
+      return mod;
+    }
+
+    public int getTypeId() {
+      return typeId;
+    }
+
+    public boolean isRecursive() {
+      return recursive;
+    }
+
+    public boolean hasPacker() {
+      return packerProc != null && !packerProc.isNil();
+    }
+
+    public boolean hasUnpacker() {
+      return unpackerProc != null && !unpackerProc.isNil();
+    }
+
+    public IRubyObject getPackerProc() {
+      return packerProc;
+    }
+
+    public IRubyObject getUnpackerProc() {
+      return unpackerProc;
+    }
+
+    public RubyArray<?> toPackerTuple(ThreadContext ctx) {
+      return ctx.runtime.newArray(new IRubyObject[] {ctx.runtime.newFixnum(typeId), packerProc});
+    }
+
+    public RubyArray<?> toUnpackerTuple(ThreadContext ctx) {
+      return ctx.runtime.newArray(new IRubyObject[] {mod, unpackerProc});
+    }
+
+    public IRubyObject[] toPackerProcTypeIdPair(ThreadContext ctx) {
+      return new IRubyObject[] {packerProc, ctx.runtime.newFixnum(typeId)};
+    }
+  }
+}