msgpack 1.4.0.pre1 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dce37069bf08ffa0b29a76363cee45e3d8e19188236d673dc1c9b4172727738c
-  data.tar.gz: 3c30d3ac0b3a3b4f5432e958935370f2e55d16ce516bf6a91eb12382aad38d8b
+  metadata.gz: 451ed743fe37bd3726f897f4daae9e7fd9b2cb065a34cf438ca786350060e0ec
+  data.tar.gz: 87335c4b1d35f9ac88af648963b1fe60caa72059ec7e6c1b6b507027da9118d8
 SHA512:
-  metadata.gz: 53d062f57dcd7324a04224c16dedbe7aa0188515d595f5a38ca543e0b956db8bbd3906a59cf3faceb5c0fb3b23d10be47bb34d19146c61923818f8f856c0a726
-  data.tar.gz: 408bcb8f9cde8c88ac6a0620a8a64c746e34723b582aa160b53687f7b5206a9cdcf97efb97eafcf6357cee6412ae3d0346a952e97b979f7c4eeffe23150d7ec4
+  metadata.gz: eb519049e9d8e4b36682e4b2c2625f10172745ce74ad1cfc8e6d93468b219071ed579bcfd36b7f0e1210eeb177197b435da2a028f46d2c71c73a3bcf2b53dcf8
+  data.tar.gz: e8c4765ac836226f5a2ce68a62194f112fc77bcdd9744d39040ff2db5a2c27d876348fb6002110c00b122bccdff2d539223d82ff73ea0bcb4cf1a3d9fb62defe

data/ChangeLog

@@ -1,3 +1,17 @@
+2021-02-01 version 1.4.2:
+
+* Add the required Ruby version (>= 2.4) to avoid compilation errors on older Ruby runtimes
+* Drop the support of old Ruby versions explicitly (1.8, 1.9, 2.0, 2.1, 2.2, 2.3)
+
+2021-01-27 version 1.4.1:
+
+* Bugfix about the wrong string encoding longer than 256 bytes (#200)
+
+2021-01-27 version 1.4.0:
+
+* Introduce the optimization to use frozen/deduped keys for map objects
+* Stop releasing fat gem (pre-built binaries) for mswin32 arch environments
+
 2020-02-05 version 1.3.3:
 
 * Hotfix release for Windows environments: 1.3.2 missed including binaries

data/README.md

@@ -0,0 +1,242 @@
+# MessagePack
+
+[MessagePack](http://msgpack.org) is an efficient binary serialization format.
+It lets you exchange data among multiple languages like JSON but it's faster and smaller.
+For example, small integers (like flags or error code) are encoded into a single byte,
+and typical short strings only require an extra byte in addition to the strings themselves.
+
+If you ever wished to use JSON for convenience (storing an image with metadata) but could
+not for technical reasons (binary data, size, speed...), MessagePack is a perfect replacement.
+
+    require 'msgpack'
+    msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+    MessagePack.unpack(msg)   #=> [1,2,3]
+
+Use RubyGems to install:
+
+    gem install msgpack
+
+or build msgpack-ruby and install:
+
+    bundle
+    rake
+    gem install --local pkg/msgpack
+
+
+## Use cases
+
+* Create REST API returing MessagePack using Rails + [RABL](https://github.com/nesquena/rabl)
+* Store objects efficiently serialized by msgpack on memcached or Redis
+  * In fact Redis supports msgpack in [EVAL-scripts](http://redis.io/commands/eval)
+* Upload data in efficient format from mobile devices such as smartphones
+  * MessagePack works on iPhone/iPad and Android. See also [Objective-C](https://github.com/msgpack/msgpack-objectivec) and [Java](https://github.com/msgpack/msgpack-java) implementations
+* Design a portable protocol to communicate with embedded devices
+  * Check also [Fluentd](http://fluentd.org/) which is a log collector which uses msgpack for the log format (they say it uses JSON but actually it's msgpack, which is compatible with JSON)
+* Exchange objects between software components written in different languages
+  * You'll need a flexible but efficient format so that components exchange objects while keeping compatibility
+
+## Portability
+
+MessagePack for Ruby should run on x86, ARM, PowerPC, SPARC and other CPU architectures.
+
+And it works with MRI (CRuby) and Rubinius.
+Patches to improve portability is highly welcomed.
+
+
+## Serializing objects
+
+Use `MessagePack.pack` or `to_msgpack`:
+
+```ruby
+require 'msgpack'
+msg = MessagePack.pack(obj)  # or
+msg = obj.to_msgpack
+```
+
+### Streaming serialization
+
+Packer provides advanced API to serialize objects in streaming style:
+
+```ruby
+# serialize a 2-element array [e1, e2]
+pk = MessagePack::Packer.new(io)
+pk.write_array_header(2).write(e1).write(e2).flush
+```
+
+See [API reference](http://ruby.msgpack.org/MessagePack/Packer.html) for details.
+
+## Deserializing objects
+
+Use `MessagePack.unpack`:
+
+```ruby
+require 'msgpack'
+obj = MessagePack.unpack(msg)
+```
+
+### Streaming deserialization
+
+Unpacker provides advanced API to deserialize objects in streaming style:
+
+```ruby
+# deserialize objects from an IO
+u = MessagePack::Unpacker.new(io)
+u.each do |obj|
+  # ...
+end
+```
+
+or event-driven style which works well with EventMachine:
+
+```ruby
+# event-driven deserialization
+def on_read(data)
+  @u ||= MessagePack::Unpacker.new
+  @u.feed_each(data) {|obj|
+     # ...
+  }
+end
+```
+
+See [API reference](http://ruby.msgpack.org/MessagePack/Unpacker.html) for details.
+
+## Serializing and deserializing symbols
+
+By default, symbols are serialized as strings:
+
+```ruby
+packed = :symbol.to_msgpack     # => "\xA6symbol"
+MessagePack.unpack(packed)      # => "symbol"
+```
+
+This can be customized by registering an extension type for them:
+
+```ruby
+MessagePack::DefaultFactory.register_type(0x00, Symbol)
+
+# symbols now survive round trips
+packed = :symbol.to_msgpack     # => "\xc7\x06\x00symbol"
+MessagePack.unpack(packed)      # => :symbol
+```
+
+The extension type for symbols is configurable like any other extension type.
+For example, to customize how symbols are packed you can just redefine
+Symbol#to_msgpack_ext. Doing this gives you an option to prevent symbols from
+being serialized altogether by throwing an exception:
+
+```ruby
+class Symbol
+    def to_msgpack_ext
+        raise "Serialization of symbols prohibited"
+    end
+end
+
+MessagePack::DefaultFactory.register_type(0x00, Symbol)
+
+[1, :symbol, 'string'].to_msgpack  # => RuntimeError: Serialization of symbols prohibited
+```
+
+## Serializing and deserializing Time instances
+
+There are the timestamp extension type in MessagePack,
+but it is not registered by default.
+
+To map Ruby's Time to MessagePack's timestamp for the default factory:
+
+```ruby
+MessagePack::DefaultFactory.register_type(
+  MessagePack::Timestamp::TYPE, # or just -1
+  Time,
+  packer: MessagePack::Time::Packer,
+  unpacker: MessagePack::Time::Unpacker
+)
+```
+
+See [API reference](http://ruby.msgpack.org/) for details.
+
+## Extension Types
+
+Packer and Unpacker support [Extension types of MessagePack](https://github.com/msgpack/msgpack/blob/master/spec.md#types-extension-type).
+
+```ruby
+# register how to serialize custom class at first
+pk = MessagePack::Packer.new(io)
+pk.register_type(0x01, MyClass1, :to_msgpack_ext) # equal to pk.register_type(0x01, MyClass)
+pk.register_type(0x02, MyClass2){|obj| obj.how_to_serialize() } # blocks also available
+
+# almost same API for unpacker
+uk = MessagePack::Unpacker.new()
+uk.register_type(0x01, MyClass1, :from_msgpack_ext)
+uk.register_type(0x02){|data| MyClass2.create_from_serialized_data(data) }
+```
+
+`MessagePack::Factory` is to create packer and unpacker which have same extension types.
+
+```ruby
+factory = MessagePack::Factory.new
+factory.register_type(0x01, MyClass1) # same with next line
+factory.register_type(0x01, MyClass1, packer: :to_msgpack_ext, unpacker: :from_msgpack_ext)
+pk = factory.packer(options_for_packer)
+uk = factory.unpacker(options_for_unpacker)
+```
+
+For `MessagePack.pack` and `MessagePack.unpack`, default packer/unpacker refer `MessagePack::DefaultFactory`. Call `MessagePack::DefaultFactory.register_type` to enable types process globally.
+
+```ruby
+MessagePack::DefaultFactory.register_type(0x03, MyClass3)
+MessagePack.unpack(data_with_ext_typeid_03) #=> MyClass3 instance
+```
+
+## Buffer API
+
+MessagePack for Ruby provides a buffer API so that you can read or write data by hand, not via Packer or Unpacker API.
+
+This [MessagePack::Buffer](http://ruby.msgpack.org/MessagePack/Buffer.html) is backed with a fixed-length shared memory pool which is very fast for small data (<= 4KB),
+and has zero-copy capability which significantly affects performance to handle large binary data.
+
+## How to build and run tests
+
+Before building msgpack, you need to install bundler and dependencies.
+
+    gem install bundler
+    bundle install
+
+Then, you can run the tasks as follows:
+
+### Build
+
+    bundle exec rake build
+
+### Run tests
+
+    bundle exec rake spec
+
+### Generating docs
+
+    bundle exec rake doc
+
+## How to build -java rubygems
+
+To build -java gems for JRuby, run:
+
+    rake build:java
+
+If this directory has Gemfile.lock (generated with MRI), remove it beforehand.
+
+## Updating documents
+
+Online documents (http://ruby.msgpack.org) is generated from gh-pages branch.
+Following commands update documents in gh-pages branch:
+
+    bundle exec rake doc
+    git checkout gh-pages
+    cp doc/* ./ -a
+
+## Copyright
+
+* Author
+  * Sadayuki Furuhashi <frsyuki@gmail.com>
+* Copyright
+  * Copyright (c) 2008-2015 Sadayuki Furuhashi
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -64,13 +64,6 @@ end
 namespace :build do
   desc 'Build gem for JRuby after cleaning'
   task :java => [:clean, :spec, :build]
-
-  desc 'Build gems for Windows per rake-compiler-dock'
-  task :windows do
-    require 'rake_compiler_dock'
-    # See RUBY_CC_VERSION in https://github.com/rake-compiler/rake-compiler-dock/blob/master/Dockerfile
-    RakeCompilerDock.sh 'bundle && gem i json && rake cross native gem RUBY_CC_VERSION=2.2.2:2.3.0:2.4.0:2.5.0:2.6.0:2.7.0'
-  end
 end
 
 CLEAN.include('lib/msgpack/msgpack.*')

data/ext/java/org/msgpack/jruby/Decoder.java

@@ -38,36 +38,38 @@ public class Decoder implements Iterator<IRubyObject> {
   private ExtensionRegistry registry;
   private ByteBuffer buffer;
   private boolean symbolizeKeys;
+  private boolean freeze;
   private boolean allowUnknownExt;
 
   public Decoder(Ruby runtime) {
-    this(runtime, null, new byte[] {}, 0, 0, false, false);
+    this(runtime, null, new byte[] {}, 0, 0, false, false, false);
   }
 
   public Decoder(Ruby runtime, ExtensionRegistry registry) {
-    this(runtime, registry, new byte[] {}, 0, 0, false, false);
+    this(runtime, registry, new byte[] {}, 0, 0, false, false, false);
   }
 
   public Decoder(Ruby runtime, byte[] bytes) {
-    this(runtime, null, bytes, 0, bytes.length, false, false);
+    this(runtime, null, bytes, 0, bytes.length, false, false, false);
   }
 
   public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes) {
-    this(runtime, registry, bytes, 0, bytes.length, false, false);
+    this(runtime, registry, bytes, 0, bytes.length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean allowUnknownExt) {
-    this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, allowUnknownExt);
+  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
+    this(runtime, registry, bytes, 0, bytes.length, symbolizeKeys, freeze, allowUnknownExt);
   }
 
   public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length) {
-    this(runtime, registry, bytes, offset, length, false, false);
+    this(runtime, registry, bytes, offset, length, false, false, false);
   }
 
-  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean allowUnknownExt) {
+  public Decoder(Ruby runtime, ExtensionRegistry registry, byte[] bytes, int offset, int length, boolean symbolizeKeys, boolean freeze, boolean allowUnknownExt) {
     this.runtime = runtime;
     this.registry = registry;
     this.symbolizeKeys = symbolizeKeys;
+    this.freeze = freeze;
     this.allowUnknownExt = allowUnknownExt;
     this.binaryEncoding = runtime.getEncodingService().getAscii8bitEncoding();
     this.utf8Encoding = UTF8Encoding.INSTANCE;
@@ -118,7 +120,12 @@ public class Decoder implements Iterator<IRubyObject> {
   private IRubyObject consumeString(int size, Encoding encoding) {
     byte[] bytes = readBytes(size);
     ByteList byteList = new ByteList(bytes, encoding);
-    return runtime.newString(byteList);
+    RubyString string = runtime.newString(byteList);
+    if (this.freeze) {
+      string.setFrozen(true);
+      string = runtime.freezeAndDedupString(string);
+    }
+    return string;
   }
 
   private IRubyObject consumeArray(int size) {
@@ -220,6 +227,14 @@ public class Decoder implements Iterator<IRubyObject> {
 
   @Override
   public IRubyObject next() {
+    IRubyObject next = consumeNext();
+    if (freeze) {
+      next.setFrozen(true);
+    }
+    return next;
+  }
+
+  private IRubyObject consumeNext() {
     int position = buffer.position();
     try {
       byte b = buffer.get();

data/ext/java/org/msgpack/jruby/Unpacker.java

@@ -34,6 +34,7 @@ public class Unpacker extends RubyObject {
   private Decoder decoder;
   private final RubyClass underflowErrorClass;
   private boolean symbolizeKeys;
+  private boolean freeze;
   private boolean allowUnknownExt;
 
   public Unpacker(Ruby runtime, RubyClass type) {
@@ -56,6 +57,7 @@ public class Unpacker extends RubyObject {
   public IRubyObject initialize(ThreadContext ctx, IRubyObject[] args) {
     symbolizeKeys = false;
     allowUnknownExt = false;
+    freeze = false;
     if (args.length > 0) {
       if (args[args.length - 1] instanceof RubyHash) {
         RubyHash options = (RubyHash) args[args.length - 1];
@@ -63,6 +65,10 @@ public class Unpacker extends RubyObject {
         if (sk != null) {
           symbolizeKeys = sk.isTrue();
         }
+        IRubyObject f = options.fastARef(ctx.getRuntime().newSymbol("freeze"));
+        if (f != null) {
+          freeze = f.isTrue();
+        }
         IRubyObject au = options.fastARef(ctx.getRuntime().newSymbol("allow_unknown_ext"));
         if (au != null) {
           allowUnknownExt = au.isTrue();
@@ -86,6 +92,11 @@ public class Unpacker extends RubyObject {
     return symbolizeKeys ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse();
   }
 
+  @JRubyMethod(name = "freeze?")
+  public IRubyObject isFreeze(ThreadContext ctx) {
+    return freeze ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse();
+  }
+
   @JRubyMethod(name = "allow_unknown_ext?")
   public IRubyObject isAllowUnknownExt(ThreadContext ctx) {
     return allowUnknownExt ? ctx.getRuntime().getTrue() : ctx.getRuntime().getFalse();
@@ -144,7 +155,7 @@ public class Unpacker extends RubyObject {
     if (limit == -1) {
       limit = byteList.length() - offset;
     }
-    Decoder decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin() + offset, limit, symbolizeKeys, allowUnknownExt);
+    Decoder decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin() + offset, limit, symbolizeKeys, freeze, allowUnknownExt);
     try {
       data = null;
       data = decoder.next();
@@ -174,7 +185,7 @@ public class Unpacker extends RubyObject {
   public IRubyObject feed(ThreadContext ctx, IRubyObject data) {
     ByteList byteList = data.asString().getByteList();
     if (decoder == null) {
-      decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, allowUnknownExt);
+      decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, freeze, allowUnknownExt);
     } else {
       decoder.feed(byteList.unsafeBytes(), byteList.begin(), byteList.length());
     }
@@ -312,7 +323,7 @@ public class Unpacker extends RubyObject {
     ByteList byteList = str.getByteList();
     this.stream = stream;
     this.decoder = null;
-    this.decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, allowUnknownExt);
+    this.decoder = new Decoder(ctx.getRuntime(), registry, byteList.unsafeBytes(), byteList.begin(), byteList.length(), symbolizeKeys, freeze, allowUnknownExt);
     return getStream(ctx);
   }
 }

data/ext/msgpack/buffer.c

@@ -23,11 +23,11 @@
 static ID s_replace;
 #endif
 
-#ifdef COMPAT_HAVE_ENCODING  /* see compat.h*/
 int msgpack_rb_encindex_utf8;
 int msgpack_rb_encindex_usascii;
 int msgpack_rb_encindex_ascii8bit;
-#endif
+
+ID s_uminus;
 
 #ifndef DISABLE_RMEM
 static msgpack_rmem_t s_rmem;
@@ -35,11 +35,11 @@ static msgpack_rmem_t s_rmem;
 
 void msgpack_buffer_static_init()
 {
-#ifdef COMPAT_HAVE_ENCODING
+    s_uminus = rb_intern("-@");
+
     msgpack_rb_encindex_utf8 = rb_utf8_encindex();
     msgpack_rb_encindex_usascii = rb_usascii_encindex();
     msgpack_rb_encindex_ascii8bit = rb_ascii8bit_encindex();
-#endif
 
 #ifndef DISABLE_RMEM
     msgpack_rmem_init(&s_rmem);
@@ -308,9 +308,7 @@ static inline void _msgpack_buffer_add_new_chunk(msgpack_buffer_t* b)
 static inline void _msgpack_buffer_append_reference(msgpack_buffer_t* b, VALUE string)
 {
     VALUE mapped_string = rb_str_dup(string);
-#ifdef COMPAT_HAVE_ENCODING
     ENCODING_SET(mapped_string, msgpack_rb_encindex_ascii8bit);
-#endif
 
     _msgpack_buffer_add_new_chunk(b);
 
@@ -337,7 +335,6 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
 
     if(b->io != Qnil) {
         msgpack_buffer_flush(b);
-#ifdef COMPAT_HAVE_ENCODING
         if (ENCODING_GET(string) == msgpack_rb_encindex_ascii8bit) {
             rb_funcall(b->io, b->io_write_all_method, 1, string);
         } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
@@ -347,10 +344,6 @@ void _msgpack_buffer_append_long_string(msgpack_buffer_t* b, VALUE string)
         } else {
             msgpack_buffer_append(b, RSTRING_PTR(string), length);
         }
-#else
-        rb_funcall(b->io, b->io_write_all_method, 1, string);
-#endif
-
     } else if(!STR_DUP_LIKELY_DOES_COPY(string)) {
         _msgpack_buffer_append_reference(b, string);