mochilo 2.0 → 2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c4c966c5decea51dbb2f53f1af2dc07a12daef8b
-  data.tar.gz: cbe013f48186036d3b4c3b4fcad9763bf9876139
+SHA256:
+  metadata.gz: df296f6ca5b95ba44fc8491a1737be6f19c058f27425487541f38db4a5336fd5
+  data.tar.gz: 1850f2005b819cb30c012ecf60fc014539878ddf35eb04ac0a59e1cb5258f2ee
 SHA512:
-  metadata.gz: 09471508db65b21a62d5aaaa602667c087245fd8cb4cd4987d2e9402d15b5dd9baedc2dfdc664f76a13fc10f1dbf583d70c3e71a567953f3936915738147d026
-  data.tar.gz: 75df812168f9ba8abcfcb8e5326c8b466b6ce1b1bd480373debde2b8886f87dc01cdcc95209079b85c5fa59340a23acf3892ed8ec1e23b2e3ea666f1859fb42a
+  metadata.gz: bafe3c03a82ea87299b27797f5ebba3fc0f1f6296c07a2f679b96d315512ff4f745d40398258cc68ada959a82ff3f99c922d6790ebde635d022a9bb521194f7b
+  data.tar.gz: 3435b0be8504e8901eefb423490d2d98c448e6d812faf0fd5b8dc002c858831f71c69896ca2cf5e7576461ff6feed56efab0c6484c53d660f7b648aad2bfe82e

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 script:
   - bundle exec rake test
+install: ./script/bootstrap
 rvm:
   - 1.9.3
   - 2.0.0

data/Gemfile

@@ -3,3 +3,7 @@ source "https://rubygems.org"
 gemspec
 
 gem "pry"
+
+group :benchmark do
+  gem "msgpack"
+end

data/docs/format-spec.md

@@ -39,3 +39,53 @@ For serializing text up to (2^32)-1 bytes.
 => ZZZZZZZZ_ZZZZZZZZ_ZZZZZZZZ_ZZZZZZZZ - a big-endian 32-bit unsigned integer which represents the legth of data.
 => type - encoding flag, a signed 8-bit signed integer
 ```
+
+#### Extensions
+
+If the encoding flag is `0xff`, the data is not a string. The first byte of the data field defines the type, and the rest of the data field is defined per type.
+The length of the enclosing ENCx type is the combined length of ext type and ext data.
+
+```
+----+-----------+----------+==========+
+... | type=0xff | ext type | ext data |
+----+-----------+----------+==========+
+```
+
+* Symbol, ext type = 0
+* Regexp, ext type = 1
+* Time, ext type = 2
+
+##### Symbol
+
+Symbols are encoded as the symbol's name, without a NUL terminator.
+
+```
+----+-----------+----------+=============+
+... | type=0xff |   0x00   | symbol name |
+----+-----------+----------+=============+
+```
+
+##### Regexp
+
+Regexps are encoded as 32-bit integer from ruby's `Regexp#options`, an 8-bit encoding type, and the text of the regexp.
+
+```
+----+-----------+----------+--------+--------+--------+--------+--------+======+
+... | type=0xff |   0x01   |XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|YYYYYYYY| data |
+----+-----------+----------+--------+--------+--------+--------+--------+======+
+XXXXXXXX_XXXXXXXX_XXXXXXXX_XXXXXXXX 32-bit unsigned options
+YYYYYYYY encoding flag
+```
+
+##### Time
+
+Time values are encoded as a 64-bit unsigned number of seconds since the epoch, a 64-bit unsigned number of usec, and a 32-bit signed seconds of offset from UTC.
+
+```
+----+-----------+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+
+... | type=0xff |   0x02   |XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|YYYYYYYY|YYYYYYYY|YYYYYYYY|YYYYYYYY|YYYYYYYY|YYYYYYYY|YYYYYYYY|YYYYYYYY|ZZZZZZZZ|ZZZZZZZZ|ZZZZZZZZ|ZZZZZZZZ|
+----+-----------+----------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+
+XXXXXXXX... whole seconds since epoch
+YYYYYYYY... usec fraction to add to X
+ZZZZZZZZ... (signed) seconds offset from UTC
+```

data/ext/mochilo/mochilo.h

@@ -149,6 +149,13 @@ enum msgpack_err_t {
 	MSGPACK_ENOTHING = -3,
 };
 
+#define MOCHILO_EXT_TYPE 0xff
+enum mochilo_ext_types_t {
+	MOCHILO_T_SYMBOL,
+	MOCHILO_T_REGEXP,
+	MOCHILO_T_TIME,
+};
+
 typedef void * mo_value;
 typedef uint64_t mo_integer;
 int mochilo_unpack_one(mo_value *_value, mochilo_src *src);

data/ext/mochilo/mochilo_api.h

@@ -4,6 +4,32 @@ MOAPI mo_value moapi_bytes_new(const char *src, size_t len)
 	return (mo_value)rb_str_new(src, len);
 }
 
+MOAPI mo_value moapi_symbol_new(const char *src, size_t len)
+{
+	return ID2SYM(rb_intern2(src, len));
+}
+
+MOAPI mo_value moapi_regexp_new(const char *src, size_t len, enum msgpack_enc_t encoding, int reg_options)
+{
+	int index = 0;
+	VALUE re;
+
+	if (encoding < sizeof(mochilo_enc_lookup)/sizeof(mochilo_enc_lookup[0]))
+		index = rb_enc_find_index(mochilo_enc_lookup[encoding]);
+
+	re = rb_reg_new(src, len, reg_options);
+	rb_enc_set_index(re, index);
+
+	return (mo_value)re;
+}
+
+MOAPI mo_value moapi_time_new(uint64_t sec, uint64_t usec, int32_t utc_offset)
+{
+	VALUE utc_time = rb_time_new(sec, usec);
+	return (mo_value)rb_funcall(utc_time,
+		rb_intern("getlocal"), 1, INT2FIX(utc_offset));
+}
+
 MOAPI mo_value moapi_str_new(const char *src, size_t len, enum msgpack_enc_t encoding)
 {
 	int index = 0;

data/ext/mochilo/mochilo_pack.c

@@ -9,6 +9,26 @@ extern VALUE rb_eMochiloPackError;
 
 void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object);
 
+static void mochilo_buf_put_ext_size(mochilo_buf *buf, long size)
+{
+	if (size < 0x100) {
+		uint8_t lead = size;
+		mochilo_buf_putc(buf, MSGPACK_T_ENC8);
+		mochilo_buf_putc(buf, lead);
+	} else if (size < 0x10000) {
+		uint16_t lead = size;
+		mochilo_buf_putc(buf, MSGPACK_T_ENC16);
+		mochilo_buf_put16be(buf, &lead);
+	} else if (size < 0x100000000) {
+		mochilo_buf_putc(buf, MSGPACK_T_ENC32);
+		mochilo_buf_put32be(buf, &size);
+	} else {
+		// there is no ext 64
+		rb_raise(rb_eMochiloPackError,
+			"String cannot be larger than %ld bytes", 0x100000000);
+	}
+}
+
 void mochilo_pack_fixnum(mochilo_buf *buf, VALUE rb_fixnum)
 {
 	long fixnum = NUM2LONG(rb_fixnum);
@@ -73,6 +93,63 @@ void mochilo_pack_bignum(mochilo_buf *buf, VALUE rb_bignum)
 	}
 }
 
+static void mochilo_put_ext_size_and_type(mochilo_buf *buf, size_t size, enum mochilo_ext_types_t type)
+{
+	mochilo_buf_put_ext_size(buf, size + 1);
+	mochilo_buf_putc(buf, MOCHILO_EXT_TYPE);
+	mochilo_buf_putc(buf, type);
+}
+
+void mochilo_pack_symbol(mochilo_buf *buf, VALUE rb_symbol)
+{
+	char *symbol_name;
+	size_t size;
+
+	symbol_name = rb_id2name(SYM2ID(rb_symbol));
+	size = strlen(symbol_name);
+
+	mochilo_put_ext_size_and_type(buf, size, MOCHILO_T_SYMBOL);
+	mochilo_buf_put(buf, symbol_name, size);
+}
+
+void mochilo_pack_regexp(mochilo_buf *buf, VALUE rb_regexp)
+{
+	uint32_t options;
+	size_t size;
+	rb_encoding *encoding;
+	char *enc_name;
+	const struct mochilo_enc_map *enc2id;
+
+	encoding = rb_enc_get(rb_regexp);
+	enc_name = rb_enc_name(encoding);
+	enc2id = mochilo_encoding_to_id(enc_name, (unsigned int)strlen(enc_name));
+
+	options = rb_reg_options(rb_regexp);
+	size = RREGEXP_SRC_LEN(rb_regexp);
+
+	mochilo_put_ext_size_and_type(buf, 4 + 1 + size, MOCHILO_T_REGEXP);
+	mochilo_buf_put32be(buf, &options);
+	mochilo_buf_putc(buf, enc2id ? enc2id->id : 0);
+	mochilo_buf_put(buf, RREGEXP_SRC_PTR(rb_regexp), size);
+}
+
+void mochilo_pack_time(mochilo_buf *buf, VALUE rb_time)
+{
+	struct time_object *tobj;
+	uint64_t sec;
+	uint64_t usec;
+	int32_t utc_offset;
+
+	sec = NUM2ULONG(rb_funcall(rb_time, rb_intern("to_i"), 0));
+	usec = NUM2ULONG(rb_funcall(rb_time, rb_intern("usec"), 0));
+	utc_offset = NUM2INT(rb_funcall(rb_time, rb_intern("utc_offset"), 0));
+
+	mochilo_put_ext_size_and_type(buf, 8+8+4, MOCHILO_T_TIME);
+	mochilo_buf_put64be(buf, &sec);
+	mochilo_buf_put64be(buf, &usec);
+	mochilo_buf_put32be(buf, &utc_offset);
+}
+
 struct mochilo_hash_pack {
 	mochilo_buf *buf;
 };
@@ -176,23 +253,7 @@ void mochilo_pack_str(mochilo_buf *buf, VALUE rb_str)
 		}
 	} else {
 		// if another encoding is used we need to use our custom types
-		if (size < 0x100) {
-			uint8_t lead = size;
-			mochilo_buf_putc(buf, MSGPACK_T_ENC8);
-			mochilo_buf_putc(buf, lead);
-		} else if (size < 0x10000) {
-			uint16_t lead = size;
-			mochilo_buf_putc(buf, MSGPACK_T_ENC16);
-			mochilo_buf_put16be(buf, &lead);
-		} else if (size < 0x100000000) {
-			mochilo_buf_putc(buf, MSGPACK_T_ENC32);
-			mochilo_buf_put32be(buf, &size);
-		} else {
-			// there is no ext 64
-			rb_raise(rb_eMochiloPackError,
-				"String cannot be larger than %ld bytes", 0x100000000);
-		}
-
+		mochilo_buf_put_ext_size(buf, size);
 		enc2id = mochilo_encoding_to_id(enc_name, (unsigned int)strlen(enc_name));
 		mochilo_buf_putc(buf, enc2id ? enc2id->id : 0);
 	}
@@ -267,9 +328,21 @@ void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object)
 		mochilo_pack_bignum(buf, rb_object);
 		return;
 
+	case T_SYMBOL:
+		mochilo_pack_symbol(buf, rb_object);
+		return;
+
+	case T_REGEXP:
+		mochilo_pack_regexp(buf, rb_object);
+		return;
+
 	default:
-		rb_raise(rb_eMochiloPackError,
+		if (rb_cTime == rb_obj_class(rb_object)) {
+			mochilo_pack_time(buf, rb_object);
+		} else {
+			rb_raise(rb_eMochiloPackError,
 				"Unsupported object type: %s", rb_obj_classname(rb_object));
+		}
 		return;
 	}
 }

data/ext/mochilo/mochilo_unpack.c

@@ -6,6 +6,8 @@
 #include "mochilo.h"
 #include "mochilo_api.h"
 
+extern VALUE rb_eMochiloUnpackError;
+
 static inline int unpack_array(mo_value *_array, size_t elements, mochilo_src *buf)
 {
 	size_t i;
@@ -57,6 +59,69 @@ static inline int unpack_hash(mo_value *_hash, size_t elements, mochilo_src *buf
 	return 0; \
 }
 
+static int mochilo_unpack_custom(mo_value *_value, mochilo_src *src, size_t length)
+{
+	uint8_t custom_type;
+
+	SRC_ENSURE_AVAIL(src, length);
+
+	if (length < 1)
+		return -1;
+	length--;
+	mochilo_src_get8be(src, &custom_type);
+
+	switch (custom_type) {
+		case MOCHILO_T_SYMBOL:
+		{
+			const char *ptr;
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+			*_value = moapi_symbol_new(ptr, length);
+			return 0;
+		}
+
+		case MOCHILO_T_REGEXP:
+		{
+			uint32_t options;
+			uint8_t encoding;
+			const char *ptr;
+
+			if (length < 5)
+				return -1;
+			mochilo_src_get32be(src, &options);
+			mochilo_src_get8be(src, &encoding);
+			length -= 5;
+
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+
+			*_value = moapi_regexp_new(ptr, length, encoding, options);
+			return 0;
+		}
+
+		case MOCHILO_T_TIME:
+		{
+			uint64_t sec;
+			uint64_t usec;
+			int32_t utc_offset;
+
+			if (length != 8+8+4)
+				return -1;
+
+			mochilo_src_get64be(src, &sec);
+			mochilo_src_get64be(src, &usec);
+			mochilo_src_get32be(src, &utc_offset);
+
+			*_value = moapi_time_new(sec, usec, utc_offset);
+			return 0;
+		}
+
+		default:
+			rb_raise(rb_eMochiloUnpackError, "unknown custom type 0x%02x", custom_type);
+			return -1;
+	}
+}
+
 int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 {
 	uint8_t leader;
@@ -214,6 +279,9 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			mochilo_src_get8be(src, &length);
 			mochilo_src_get8be(src, &encoding);
 
+			if (encoding == MOCHILO_EXT_TYPE)
+				return mochilo_unpack_custom(_value, src, length);
+
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 
@@ -231,6 +299,9 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			mochilo_src_get16be(src, &length);
 			mochilo_src_get8be(src, &encoding);
 
+			if (encoding == MOCHILO_EXT_TYPE)
+				return mochilo_unpack_custom(_value, src, length);
+
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 
@@ -248,6 +319,9 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			mochilo_src_get32be(src, &length);
 			mochilo_src_get8be(src, &encoding);
 
+			if (encoding == MOCHILO_EXT_TYPE)
+				return mochilo_unpack_custom(_value, src, length);
+
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 

data/lib/mochilo/version.rb

@@ -1,3 +1,3 @@
 module Mochilo
-  VERSION = "2.0"
+  VERSION = "2.1"
 end

data/mochilo.gemspec

@@ -15,10 +15,9 @@ Gem::Specification.new do |s|
   s.summary = %q{A ruby library for BananaPack}
   s.test_files = `git ls-files spec`.split("\n")
   s.required_ruby_version = ">= 1.9.3"
+  s.licenses = ["MIT"]
 
   # tests
   s.add_development_dependency 'rake-compiler', ">= 0.8.1"
   s.add_development_dependency 'minitest', ">= 4.1.0"
-  # benchmarks
-  s.add_development_dependency 'msgpack'
 end

data/script/bootstrap

@@ -12,5 +12,6 @@ cd "$(dirname "$0")/.."
 if bundle check 1>/dev/null 2>&1; then
     echo "Gem environment up-to-date"
 else
-    exec bundle install --binstubs --path vendor/gems "$@"
+    gem update bundler
+    exec bundle install --binstubs --path vendor/gems --without benchmark "$@"
 fi

data/test/pack_test.rb

@@ -276,9 +276,32 @@ class MochiloPackTest < Minitest::Test
     end
   end
 
-  def test_pack_symbol_fails
-    assert_raises Mochilo::PackError do
-      Mochilo.pack(:symbol)
+  def test_pack_symbol
+    expected = "\xC7\x07\xFF\x00symbol"
+    assert_equal expected, Mochilo.pack(:symbol)
+    assert_equal :symbol, Mochilo.unpack(expected)
+  end
+
+  def test_pack_regexp
+    expected = "\xC7\x0D\xFF\x01\x00\x00\x00\x00\x01pa.tern"
+    assert_equal expected, Mochilo.pack(/pa.tern/)
+    [
+      /pa.tern/,
+      /thing/im,
+    ].each do |re|
+      assert_equal re, Mochilo.unpack(Mochilo.pack(re))
     end
   end
+
+  def test_time
+    offset = -13*60*60 # I don't know if this is possible. There shouldn't be anything with a greater absolute value.
+    t = Time.gm(2042, 7, 21, 3, 32, 37, 974010).getlocal(offset)
+    expected = "\xC7\x15\xFF\x02" +
+      "\x00\x00\x00\x00\x88\x77\x66\x55" + # sec
+      "\x00\x00\x00\x00\x00\x0E\xDC\xBA" + # usec
+      "\xFF\xFF\x49\x30"                   # utc_offset
+    assert_equal expected, Mochilo.pack(t)
+    assert_equal t, Mochilo.unpack(expected)
+    assert_equal offset, Mochilo.unpack(expected).utc_offset
+  end
 end

data/test/setup.rb

@@ -10,6 +10,10 @@ require 'bundler/setup'
 # bring in minitest
 require 'minitest/autorun'
 
+if !defined?(MiniTest::Test)
+  MiniTest::Test = MiniTest::Unit::TestCase
+end
+
 # put lib and test dirs directly on load path
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 $LOAD_PATH.unshift File.expand_path('..', __FILE__)

data/test/unpack_test.rb

@@ -314,4 +314,41 @@ class MochiloUnpackTest < Minitest::Test
     # arr.each {|i| data << (Mochilo.pack(i) + Mochilo.pack(i))}
     # assert_equal hash, Mochilo.unpack(data)
   end
+
+  def test_unpack_custom_types_with_errors
+    # regexp has all the options bits
+    re = Mochilo.unpack("\xC7\x0D\xFF\x01\xFF\xFF\xFF\xFF\x01pa.tern")
+    refute_equal 0, re.options, "unpacked regexp options"
+
+    # time is missing the last char
+    assert_raises Mochilo::UnpackError do
+      Mochilo.unpack("\xC7\x11\xFF\x02\x00\x00\x00\x00\x88\x77\x66\x55\x00\x00\x00\x00\x00\x0E\xDC")
+    end
+
+    # time overflows
+    assert_raises RangeError do
+      # set the top bit of the time.....v ...and of usec................v
+      Mochilo.unpack("\xC7\x15\xFF\x02\x80\x00\x00\x00\x88\x77\x66\x55\x80\x00\x00\x00\x00\x0E\xDC\xBA\xFF\xFF\xFF\xFF")
+    end
+
+    # unknown field
+    assert_raises Mochilo::UnpackError do
+      Mochilo.unpack("\xC7\x07\xFF\x10unknowncustomtype")
+    end
+  end
+
+  # Make sure that the custom parsers never read past the end
+  # of their buffers.
+  def test_unpack_custom_types_with_missing_char
+    assert_error_on_truncated_unpack "\xC7\x07\xFF\x00symbol"
+    assert_error_on_truncated_unpack "\xC7\x0D\xFF\x01\x00\x00\x00\x00\x01pa.tern"
+    assert_error_on_truncated_unpack "\xC7\x15\xFF\x02\x00\x00\x00\x00\x88\x77\x66\x55\x00\x00\x00\x00\x00\x0E\xDC\xBA\x00\x00\x00\x00"
+  end
+
+  def assert_error_on_truncated_unpack(packed)
+    refute_nil Mochilo.unpack(packed)
+    assert_raises Mochilo::UnpackError do
+      Mochilo.unpack(packed.chop)
+    end
+  end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: mochilo
 version: !ruby/object:Gem::Version
-  version: '2.0'
+  version: '2.1'
 platform: ruby
 authors:
 - Vicent Martí
 - Brian Lopez
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-25 00:00:00.000000000 Z
+date: 2023-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -39,21 +39,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.1.0
-- !ruby/object:Gem::Dependency
-  name: msgpack
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
+description:
 email: vicent@github.com seniorlopez@gmail.com
 executables: []
 extensions:
@@ -92,9 +78,10 @@ files:
 - test/setup.rb
 - test/unpack_test.rb
 homepage: http://github.com/brianmario/mochilo
-licenses: []
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--charset=UTF-8"
 require_paths:
@@ -110,9 +97,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.3.10
+signing_key:
 specification_version: 4
 summary: A ruby library for BananaPack
 test_files: []