mochilo 1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OTMzOGM2MTMxMWI0YjZlNjZlMzhjNGU2ZDUyOWJmMWQyNGQ1OWJmZg==
+  data.tar.gz: !binary |-
+    YjdjODY4NDYyNWRhNGE2ODBiMTU4N2YwM2MwN2I0YWEyNzc0MjE0MA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MTQ5OTExMDNiMTM1MDVkODVlYTA3MDgzNGUwYzU5MGQ5OWE4MDBmY2E1Y2I5
+    NWNhOGU3MzVjOTVhYzIwZGZkMTBmYTNkZDk5Y2Y4NWI3YWE4NWEwZjljMGNk
+    ODhiY2MxNTY2ZjdlMjhmNmI2MTE5M2E1ZmY1MTExNjQ0M2Q5YTg=
+  data.tar.gz: !binary |-
+    NWNjMzNmMDU1YTM2ZGMwZGRhZWFhZDk0NzQ5NmYxNzkzMTcwNWNjOGY5OWUy
+    MTliZmJkYWU1ZGYxOThlNzJhMjc4NDAzNDY2M2E0ZGNhZmM1Y2YzMzA5ODFi
+    OGYwZDA4YTQzNmNiNzMxNmY1YjZmYjNmOTczYmU4MzdhOTBmNDk=

data/.gitignore

@@ -0,0 +1,9 @@
+*.dSYM
+*.so
+*.bundle
+/tmp
+/bin
+/vendor/gems
+/.bundle
+/.rbenv-version
+/vendor/cache/*.gem

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "pry"

data/Gemfile.lock

@@ -0,0 +1,30 @@
+PATH
+  remote: .
+  specs:
+    mochilo (1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.0.9)
+    method_source (0.8.1)
+    minitest (5.0.6)
+    msgpack (0.5.5)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.1.0)
+    rake-compiler (0.8.3)
+      rake
+    slop (3.4.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest (>= 4.1.0)
+  mochilo!
+  msgpack
+  pry
+  rake-compiler (>= 0.8.1)

data/README.md

@@ -0,0 +1,56 @@
+# Mochilo
+
+Mochilo is a Ruby library implementing the BananaPack protocol. BananaPack is a superset of MessagePack. It adds three new types to the protocol: Symbol and String in 16 and 32 bit lengths.
+
+The Symbol type is a String of text composed of ASCII characters with a maximum length of (2^8)-1 bytes.
+
+The String16 and String32 types are exactly the same as the [Raw16](http://wiki.msgpack.org/display/MSGPACK/Format+specification#Formatspecification-raw16) and [Raw32](http://wiki.msgpack.org/display/MSGPACK/Format+specification#Formatspecification-raw32) types from MessagePack except there is an encoding flag stored along with the bytes. This allows the format to differentiate binary from text.
+
+Check out docs/format-spec.md for more detailed information on the differences between BananaPack and MessagePack.
+
+## Usage
+
+``` ruby
+require 'mochilo'
+obj = {key: "value"}
+bpack = Mochilo.pack(obj)
+#=> "\x81\xD8\x00\x03\x01key\xD8\x00\x05\x00value"
+
+hash = Mochilo.unpack(bpack)
+#=> {"key"=>"value"}
+```
+
+Notice how `key` came back into Ruby as a String instead of a Symbol? This is because the `pack` method of Mochilo only generates "safe" bpack.
+
+bpack without Symbols is considered "safe" for Ruby because in Ruby, Symbols aren't garbage collected. So if you unpack arbitrary bpack from an untrusted or malicious source, you could end up potentially exhausting the memory of your server.
+
+To generate "unsafe" bpack, use `pack_unsafe` and `unpack_unsafe` methods instead:
+
+``` ruby
+require 'mochilo'
+obj = {key: "value"}
+bpack = Mochilo.pack_unsafe(obj)
+#=> "\x81\xD4\x03key\xD8\x00\x05\x00value"
+
+hash = Mochilo.unpack_unsafe(bpack)
+#=> {:key=>"value"}
+```
+
+If you attempt to unpack "unsafe" bpack without using `unpack_unsafe`, an exception is raised.
+
+## Supported Ruby Types
+
+The following Ruby types are supported. Meaning they will be deserialized into the same Ruby type they were before serialization.
+
+If any other object type is encountered during serialization, an exception is raised. This is to ensure you have explicit control over what is being serialized.
+
+* Fixnum
+* Bignum
+* Float
+* Symbol
+* String (with encoding)
+* nil
+* true
+* false
+* Array
+* Hash

data/Rakefile

@@ -0,0 +1,33 @@
+require 'date'
+require 'rake/clean'
+require 'rake/extensiontask'
+require 'digest/md5'
+
+task :default => :compile
+
+# ==========================================================
+# Ruby Extension
+# ==========================================================
+Rake::ExtensionTask.new('mochilo') do |ext|
+  ext.cross_compile = true
+  ext.cross_platform = ['x86-mingw32', 'x86-mswin32-60']
+
+  ext.lib_dir = File.join 'lib', 'mochilo'
+end
+
+desc "Open an irb session preloaded with Mochilo"
+task :console do
+  sh "irb -rubygems -I lib -r ./lib/mochilo"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new('test') do |t|
+  t.test_files = FileList['test/*_test.rb']
+  t.ruby_opts += ['-rubygems'] if defined? Gem
+end
+task 'test' => [:compile]
+
+
+task :encodings do
+  sh "ruby genperf.rb | gperf > ./ext/mochilo/encodings.h"
+end

data/docs/format-spec.md

@@ -0,0 +1,46 @@
+## String
+
+The String type is nearly identical to the Raw type from MessagePack but is specifically
+aimed at marking a set of bytes as text.
+
+### Format Specification
+
+#### Symbol
+
+For storing symbol names as ASCII text up to (2^8)-1 bytes.
+Length is stored in unsigned 8-bit integer.
+
+```
++--------+--------+----------
+|  0xd4  |XXXXXXXX|...N bytes
++--------+--------+----------
+=> XXXXXXXX (=N) bytes of raw bytes.
+```
+
+#### String16
+
+For storing text up to (2^16)-1 bytes.
+Length is stored in unsigned 16-bit big-endian integer.
+Encoding is stored as an int8
+
+```
++--------+--------+--------+--------+----------
+|  0xd8  |XXXXXXXX|XXXXXXXX|YYYYYYYY|...N bytes
++--------+--------+--------+--------+----------
+=> XXXXXXXX_XXXXXXXX (=N) bytes of raw bytes.
+=> YYYYYYYY encoding flag
+```
+
+#### String32
+
+For storing text up to (2^32)-1 bytes.
+Length is stored in unsigned 32-bit big-endian integer.
+Encoding is stored as an int8
+
+```
++--------+--------+--------+--------+--------+--------+----------
+|  0xd9  |XXXXXXXX|XXXXXXXX|XXXXXXXX|XXXXXXXX|YYYYYYYY|...N bytes
++--------+--------+--------+--------+--------+--------+----------
+=> XXXXXXXX_XXXXXXXX_XXXXXXXX_XXXXXXXX (=N) bytes of raw bytes.
+=> YYYYYYYY encoding flag
+```

data/docs/why-banana-pack.md

@@ -0,0 +1,25 @@
+# Why BananaPack?
+
+It's actually quite simple. We need to be able to serialize data while being able to differentiate between text and binary.
+
+There are many, many other serialization formats out there. Some require a schema be defined up-front, while others can be parsed  with no prior knowledge of the structure of the serialized data. Some are extremely verbose and harder to parse while others are compact and efficient to parse.
+
+A couple of them that are comparable to BananaPack are MessagePack and JSON. To help explain why we created BananaPack, let's answer a few questions about those two first.
+
+## Why not JSON?
+
+JSON has the basic principles of a serialized format that we want. It's only capable of representing primitive data types in common with almost every language. That makes it great for cross-language communication.
+
+The problem is that it's just text. Unicode text to be precise. That means you can't serialize arbitrary binary data without encoding it first. Common practice is to Base64 encode the data first. But then deserialization side *has* to know to unencode it after parsing. There's no need for hacks like this.
+
+## Why not MessagePack?
+
+MessagePack is nearly perfect for what we want. But text and binary share a single type in the spec, "Raw". That means when deserializing the data you could be reading a contents of a Markdown file, or the contents of a JPEG. There are countless hacks to get around this but there shouldn't be a need for them either.
+
+## BananaPack
+
+We wanted something compact and easy to parse. But it also needed to have native support for differentiating between text and binary data.
+
+Knowing what encoding text is in is just as important as knowing the timezone of a timestamp. This has been true since the birth of the Internet.
+
+So we used a couple of the "reserved" data type slots in the MessagePack spec to add a String16, String32 and Symbol type. String16 and String32 are exactly the same as the Raw16 and Raw32 types from the MessagePack spec except they have an encoding flag as well. This means that it's possible to serialize text data without hacks. This also means it's possible to serialize the contents of a JPEG without first needing to encode or escape it.

data/ext/mochilo/buffer.c

@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2012 GitHub, Inc
+ */
+#include <stdarg.h>
+#include <ctype.h>
+#include "mochilo.h"
+
+#define MOCHILO_CHUNK_SIZE 1024
+#define MOCHILO_CHUNK_INIT 8
+
+static mochilo_buf_chunk *init_cur_chunk(mochilo_buf *buf, size_t chunk_size)
+{
+	mochilo_buf_chunk *chunk = &buf->chunks[buf->cur_chunk];
+
+	if (chunk_size <= MOCHILO_CHUNK_SIZE)
+		chunk_size = MOCHILO_CHUNK_SIZE;
+	else
+		chunk_size += MOCHILO_CHUNK_SIZE;
+
+	buf->last_alloc = chunk->ptr = malloc(chunk_size);
+	if (!chunk->ptr)
+		return NULL;
+
+	chunk->end = chunk->ptr + chunk_size;
+	return chunk;
+}
+
+static void skip_last_chunk(mochilo_buf *buf)
+{
+	mochilo_buf_chunk *chunk = &buf->chunks[buf->cur_chunk];
+
+	buf->total_size += (chunk->ptr - buf->last_alloc);
+	buf->cur_chunk++;
+
+	chunk->end = chunk->ptr;
+	chunk->ptr = buf->last_alloc;
+}
+
+static void free_buf(mochilo_buf *buf)
+{
+	uint16_t i;
+
+	for (i = 0; i < buf->cur_chunk; ++i)
+		free(buf->chunks[i].ptr);
+
+	free(buf->chunks);
+}
+
+
+void mochilo_buf_init(mochilo_buf *buf)
+{
+	buf->chunks = malloc(MOCHILO_CHUNK_INIT * sizeof(mochilo_buf_chunk));
+	buf->total_size = 0;
+	buf->cur_chunk = 0;
+	buf->chunk_count = MOCHILO_CHUNK_INIT;
+
+	init_cur_chunk(buf, MOCHILO_CHUNK_SIZE);
+}
+
+VALUE mochilo_buf_flush(mochilo_buf *buf)
+{
+	VALUE rb_str;
+	char *ptr;
+	uint16_t i;
+
+	skip_last_chunk(buf);
+
+#ifdef RUBINIUS
+	char *alloc;
+	alloc = ptr = malloc(buf->total_size);
+	if (alloc == NULL)
+		rb_raise(rb_eNoMemError, "Failed to alloc temp buffer");
+#else
+	rb_str = rb_str_new(NULL, buf->total_size);
+	ptr = RSTRING_PTR(rb_str);
+#endif
+
+	for (i = 0; i < buf->cur_chunk; ++i) {
+		mochilo_buf_chunk *chunk = &buf->chunks[i];
+		size_t chunk_len = chunk->end - chunk->ptr;
+
+		memcpy(ptr, chunk->ptr, chunk_len);
+		ptr += chunk_len;
+		free(chunk->ptr);
+	}
+
+	free(buf->chunks);
+
+#ifdef RUBINIUS
+	rb_str = rb_str_new(alloc, buf->total_size);
+#endif
+
+	return rb_str;
+}
+
+mochilo_buf_chunk *mochilo_buf_rechunk2(mochilo_buf *buf, size_t chunk_size)
+{
+	skip_last_chunk(buf);
+
+	if (buf->cur_chunk == buf->chunk_count) {
+		buf->chunk_count *= 2;
+
+		buf->chunks = realloc(buf->chunks, buf->chunk_count * sizeof(mochilo_buf_chunk));
+		if (!buf->chunks)
+			return NULL;
+
+	}
+
+	return init_cur_chunk(buf, chunk_size);
+}
+
+mochilo_buf_chunk *mochilo_buf_rechunk(mochilo_buf *buf)
+{
+	return mochilo_buf_rechunk2(buf, MOCHILO_CHUNK_SIZE);
+}
+
+void mochilo_buf_put(mochilo_buf *buf, const char *data, size_t len)
+{
+	mochilo_buf_chunk *chunk = &buf->chunks[buf->cur_chunk];
+
+	if (unlikely(chunk->ptr + len > chunk->end)) {
+		if (!(chunk = mochilo_buf_rechunk2(buf, len)))
+			return;
+	}
+
+	memmove(chunk->ptr, data, len);
+	chunk->ptr += len;
+}
+
+const char *mochilo_src_peek(mochilo_src *buf, size_t need)
+{
+	const char *ptr;
+
+	if (unlikely(buf->ptr + need > buf->end))
+		return NULL;
+
+	ptr = buf->ptr;
+	buf->ptr += need;
+	return ptr;
+}
+

data/ext/mochilo/buffer.h

@@ -0,0 +1,146 @@
+#ifndef INCLUDE_buffer_h__
+#define INCLUDE_buffer_h__
+
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <stdint.h>
+
+#define unlikely(x)     __builtin_expect((x),0)
+
+/**
+ * Byteswap code
+ */
+static inline void swap8(const uint8_t *buffer, void *out)
+{
+	unsigned char *ptr = (unsigned char *)out;
+	*ptr = *buffer;
+}
+
+static inline void swap16(const uint8_t *buffer, void *out)
+{
+    unsigned char *ptr = (unsigned char *)out;
+    ptr[0] = buffer[1];
+    ptr[1] = buffer[0];
+}
+
+static inline void swap32(const uint8_t *buffer, void *out)
+{
+    unsigned char *ptr = (unsigned char *)out;
+    ptr[0] = buffer[3];
+    ptr[1] = buffer[2];
+    ptr[2] = buffer[1];
+    ptr[3] = buffer[0];
+}
+
+static inline void swap64(const uint8_t *buffer, void *out)
+{
+	unsigned char *ptr = (unsigned char *)out;
+	ptr[0] = buffer[7];
+	ptr[1] = buffer[6];
+	ptr[2] = buffer[5];
+	ptr[3] = buffer[4];
+	ptr[4] = buffer[3];
+	ptr[5] = buffer[2];
+	ptr[6] = buffer[1];
+	ptr[7] = buffer[0];
+}
+
+/**
+ * Buffer code 
+ */
+typedef struct {
+	char *ptr;
+	char *end;
+} mochilo_buf_chunk;
+
+typedef struct {
+	mochilo_buf_chunk *chunks;
+	char *last_alloc;
+	size_t total_size;
+	uint16_t chunk_count, cur_chunk;
+} mochilo_buf;
+
+typedef struct {
+	const char *ptr;
+	const char *end;
+	int trusted;
+} mochilo_src;
+
+void mochilo_buf_init(mochilo_buf *buf);
+VALUE mochilo_buf_flush(mochilo_buf *buf);
+
+mochilo_buf_chunk *mochilo_buf_rechunk(mochilo_buf *buf);
+mochilo_buf_chunk *mochilo_buf_rechunk2(mochilo_buf *buf, size_t chunk_size);
+
+void mochilo_buf_put(mochilo_buf *buf, const char *data, size_t len);
+
+const char *mochilo_src_peek(mochilo_src *buf, size_t need);
+
+#define BUF_ENSURE_AVAIL(b, d) \
+	mochilo_buf_chunk *chunk = &b->chunks[b->cur_chunk]; \
+	if (unlikely(chunk->ptr + (d) > chunk->end)) { \
+		if ((chunk = mochilo_buf_rechunk(b)) == NULL) return; };
+
+#define SRC_CHECK_AVAIL(src, bytes) (src->ptr + bytes <= src->end) 
+
+#define SRC_ENSURE_AVAIL(src, bytes) \
+	if (unlikely(src->ptr + bytes > src->end)) \
+		return -1;
+
+static inline void mochilo_buf_putc(mochilo_buf *buf, uint8_t c)
+{
+	BUF_ENSURE_AVAIL(buf, 1);
+	*chunk->ptr = c;
+	chunk->ptr++;
+}
+
+static inline void mochilo_buf_put16be(mochilo_buf *buf, void *src16)
+{
+	BUF_ENSURE_AVAIL(buf, 2);
+	swap16(src16, chunk->ptr);
+	chunk->ptr += 2;
+}
+
+static inline void mochilo_buf_put32be(mochilo_buf *buf, void *src32)
+{
+	BUF_ENSURE_AVAIL(buf, 4);
+	swap32(src32, chunk->ptr);
+	chunk->ptr += 4;
+}
+
+static inline void mochilo_buf_put64be(mochilo_buf *buf, void *src64)
+{
+	BUF_ENSURE_AVAIL(buf, 8);
+	swap64(src64, chunk->ptr);
+	chunk->ptr += 8;
+}
+
+
+
+static inline void mochilo_src_get8be(mochilo_src *buf, uint8_t *dst8)
+{
+	*dst8 = *buf->ptr;
+	buf->ptr += 1;
+}
+
+static inline void mochilo_src_get16be(mochilo_src *buf, void *dst16)
+{
+	swap16(buf->ptr, dst16);
+	buf->ptr += 2;
+}
+
+static inline void mochilo_src_get32be(mochilo_src *buf, void *dst32)
+{
+	swap32(buf->ptr, dst32);
+	buf->ptr += 4;
+}
+
+static inline void mochilo_src_get64be(mochilo_src *buf, void *dst64)
+{
+	swap64(buf->ptr, dst64);
+	buf->ptr += 8;
+}
+
+#endif