UEL 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 20137fd4dcf93bec9b07837ec1258036ebce587ed3bf449441207c0b6e8cdd2a
+  data.tar.gz: 76291fb545ee2b209e5788f5390d3ea0e3175f54d839fe93826670692a407137
+SHA512:
+  metadata.gz: eecdc2920f87a00afdc867d36dcce4d00bc1e768a1647ee241d9f421bcd1356e59115326a6b1a28f8f26f7109ddefe37a2717dd8862bb09c2616bdfc7dbda8fe
+  data.tar.gz: 7587e3602d1858a2e7de9a29a22efb5771957f392d48f2aa98e4f064ca07a104c992b9c2db89cab30b88bdcdcda3b22cbdb4d12b5802bebf60d1813d730a6d88

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+Gemfile.lock
+
+# rspec failure tracking
+.rspec_status
+
+*.so

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,9 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7
+  - 2.6
+  - 2.5
+  - 2.4
+before_install: gem install bundler -v 2.1.4

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at matthew.b.carey@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in untitled_etf_lib.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Matthew Carey
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# UEL (Untitled ETF Lib)
+
+The other ETF lib I could find is pretty slow, so here's this one. I couldn't think of a good name.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'uel'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install uel
+
+## Usage
+
+```ruby
+require 'uel'
+
+UEL.encode(some_object)
+UEL.decode(some_string_of_etf_data)
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/swarley/uel. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/swarley/uel/blob/master/CODE_OF_CONDUCT.md).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the UEL project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/swarley/uel/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rake/extensiontask"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => [:compile, :spec]
+
+Rake::ExtensionTask.new("uel") do |ext|
+  ext.name = "uel"
+  ext.lib_dir = "lib/"
+  ext.ext_dir = "ext/uel"
+  ext.gem_spec = Gem::Specification.load('uel.gemspec')
+end

data/benchmark.rb

@@ -0,0 +1,97 @@
+require 'bert'
+require 'erlang-etf'
+require 'oj'
+require 'untitled_etf_lib'
+require 'json'
+require 'benchmark/ips'
+
+Benchmark.ips do |x|
+  etf_data = "\x83b\x00\x0009"
+  json_data = "12345"
+  raise "OJ FAIL" unless Oj.load(json_data) == 12345
+  raise "UEL FAIL" unless UEL.decode(etf_data) == 12345
+  raise "EE FAIL" unless Erlang.binary_to_term(etf_data) == 12345
+
+  #x.report("oj integer") { Oj.load json_data }
+  #x.report("json integer") { JSON.load json_data }
+  x.report("uel integer") { UEL.decode(etf_data) }
+  x.report("ee integer") { Erlang.binary_to_term(etf_data) }
+
+  x.compare!
+end
+
+Benchmark.ips do |x|
+  etf_data = "\x83FA\x9do4T~ku"
+  json_data = "123456789.123456789"
+  raise "OJ FAIL" unless Oj.load(json_data).to_f == 123456789.123456789
+  raise "UEL FAIL" unless UEL.decode(etf_data) == 123456789.123456789
+  raise "EE FAIL" unless Erlang.binary_to_term(etf_data) == 123456789.123456789
+
+  #x.report("oj float") { Oj.load json_data }
+  #x.report("json float") { JSON.load json_data }
+  x.report("uel float") { UEL.decode(etf_data) }
+  x.report("ee float") { Erlang.binary_to_term(etf_data) }
+  
+  x.compare!
+end
+
+Benchmark.ips do |x|
+  etf_data = "\x83j"
+  json_data = "[]"
+
+  raise "OJ FAIL" unless Oj.load(json_data) == []
+  raise "UEL FAIL" unless UEL.decode(etf_data) == []
+  raise "BERT FAIL" unless Erlang.binary_to_term(etf_data) == []
+
+  #x.report("oj []") { Oj.load json_data }
+  #x.report("json []") { JSON.load json_data }
+  x.report("uel nil")  { UEL.decode(etf_data) }
+  x.report("ee nil") { Erlang.binary_to_term(etf_data) }
+  x.compare!
+end
+
+Benchmark.ips do |x|
+  etf_data = "\x83n\x08\x00\xff\xff\xff\xff\xff\xff\xff\xff"
+  json_data = 0xFFFFFFFFFFFFFFFF.to_s
+  
+  raise "OJ FAIL" unless Oj.load(json_data) == 0xFFFFFFFFFFFFFFFF
+  raise "UEL FAIL" unless UEL.decode(etf_data) == 0xFFFFFFFFFFFFFFFF
+  raise "BERT FAIL" unless Erlang.binary_to_term(etf_data) == 0xFFFFFFFFFFFFFFFF
+
+  #x.report("oj big_num") { Oj.load json_data }
+  #x.report("json big_num") { JSON.load json_data }
+  x.report("uel big_num") { UEL.decode(etf_data) }
+  x.report("ee big_num") { Erlang.binary_to_term(etf_data) }
+  x.compare!
+end
+
+Benchmark.ips do |x|
+  etf_data = "\x83l\x00\x00\x00\x03a\x01F?\xf1\x99\x99\x99\x99\x99\x9an\x06\x00\x00\xa0rN\x18\tj"
+  json_data = "[1, 1.1, 10000000000000]"
+
+  raise "OJ FAIL" unless Oj.load(json_data) == [1, 1.1, 10000000000000]
+  raise "UEL FAIL" unless UEL.decode(etf_data) == [1, 1.1, 10000000000000]
+  raise "EE FAIL" unless Erlang.binary_to_term(etf_data) == [1, 1.1, 10000000000000]
+
+  x.report("oj list") { Oj.load json_data }
+  x.report("json list") { JSON.load json_data }
+  x.report("uel list") { UEL.decode(etf_data) }
+  x.report("ee list") { Erlang.binary_to_term(etf_data) }
+  x.compare!
+end
+
+Benchmark.ips do |x|
+  etf_data = "\x83t\x00\x00\x00\x02m\x00\x00\x00\x03fooa\x02m\x00\x00\x00\x03barl\x00\x00\x00\x02a\x04a\x05j"
+  json_data = '{"foo": 2, "bar": [4, 5]}'
+
+
+  raise "OJ FAIL" unless Oj.load(json_data) == {"foo" => 2, "bar" => [4, 5]}
+  raise "UEL FAIL" unless UEL.decode(etf_data) == {"foo" => 2, "bar" => [4, 5]}
+  raise "EE FAIL" unless Erlang.binary_to_term(etf_data) == {"foo" => 2, "bar" => [4, 5]}
+
+  x.report("oj map") { Oj.load json_data }
+  x.report("json map") { JSON.load json_data }
+  x.report("uel map") { UEL.decode(etf_data) }
+  x.report("ee map") { Erlang.binary_to_term(etf_data) }
+  x.compare!
+end

data/ext/uel/decode.c

@@ -0,0 +1,288 @@
+#include <netinet/in.h>
+#include <endian.h>
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <string.h>
+#include <stdlib.h>
+#include "./extconf.h"
+#include "./uel.h"
+
+#if HAVE_ZLIB_H
+#include <zlib.h>
+#endif
+
+VALUE uel_decode(VALUE self, VALUE data) {
+    struct uel_bert_data bert_data;
+ 
+    Check_Type(data, T_STRING);
+
+    bert_data.buff = (uint8_t *)RSTRING_PTR(data);
+    bert_data.end = (uint8_t *)(bert_data.buff + RSTRING_LEN(data));
+
+    if (uel_read8(&bert_data) != UEL_ETF_VERSION_NUMBER)
+        rb_raise(rb_eNotImpError, "Unknown ETF version");
+
+    return uel_decode_term(&bert_data);
+}
+
+VALUE uel_decode_term(struct uel_bert_data *data) {    
+    switch(uel_read8(data)) {
+        case UEL_ETF_SMALL_INTEGER_ID:
+            return uel_read_small_integer(data);
+        case UEL_ETF_INTEGER_ID:
+            return uel_read_integer(data);
+        case UEL_ETF_FLOAT_ID:
+            return uel_read_float(data);
+        case UEL_ETF_SMALL_BIGNUM_ID:
+            return uel_read_small_bignum(data);
+        case UEL_ETF_LARGE_BIGNUM_ID:
+            return uel_read_large_bignum(data);
+        case UEL_ETF_NEW_FLOAT_ID:
+            return uel_read_new_float(data);
+        case UEL_ETF_NIL_ID:
+            return uel_read_nil(data);
+        case UEL_ETF_LIST_ID:
+            return uel_read_list(data);
+        case UEL_ETF_MAP_ID:
+            return uel_read_map(data);
+        case UEL_ETF_BINARY_ID:
+            return uel_read_binary(data);
+        case UEL_ETF_SMALL_TUPLE_ID:
+            return uel_read_small_tuple(data);
+        case UEL_ETF_LARGE_TUPLE_ID:
+            return uel_read_large_tuple(data);
+        case UEL_ETF_ATOM_ID:
+        case UEL_ETF_ATOM_UTF8_ID:
+            return uel_read_atom(data);
+        case UEL_ETF_SMALL_ATOM_ID:
+        case UEL_ETF_SMALL_ATOM_UTF8_ID:
+            return uel_read_small_atom(data);
+#ifdef HAVE_ZLIB_H
+        case UEL_ETF_ZLIB_ID:
+            return uel_read_zlib(data);
+#endif
+        default:
+            rb_raise(rb_eNotImpError, "Unable to process data type: %hu", *(data->buff - 1));
+            return Qnil;
+    }
+}
+
+inline void uel_check_bounds(struct uel_bert_data *data, size_t size) {
+    if(data->buff + size >  data->end)
+        rb_raise(rb_eRangeError, "end of buffer while taking %li", size);
+}
+
+inline uint8_t uel_read8(struct uel_bert_data *data) {
+    uel_check_bounds(data, 1);
+    return *data->buff++;
+}
+
+inline uint16_t uel_read16(struct uel_bert_data *data) {
+    uint16_t value;
+
+    uel_check_bounds(data, 2);
+    value  = *(uint16_t *)data->buff;
+    data->buff += sizeof(uint16_t);
+    return ntohs(value);
+}
+
+inline uint32_t uel_read32(struct uel_bert_data *data) {
+    uint32_t value;
+
+    uel_check_bounds(data, 4);
+    value =  *(uint32_t *)data->buff;
+    data->buff += sizeof(uint32_t);
+    return ntohl(value);
+}
+
+inline VALUE uel_read_small_integer(struct uel_bert_data *data) {
+    return UINT2NUM((uint8_t) uel_read8(data));
+}
+
+inline VALUE uel_read_integer(struct uel_bert_data *data) {
+    return INT2NUM(uel_read32(data));
+}
+
+VALUE uel_read_float(struct uel_bert_data *data) {
+    double value;
+
+    uel_check_bounds(data, 31);
+    sscanf((char *)data->buff, "%31lf", &value);
+    data->buff += 31;
+    return DBL2NUM(value);
+}
+
+inline VALUE uel_read_nil(struct uel_bert_data *data) {
+    return rb_ary_new();
+}
+
+VALUE uel_read_small_bignum(struct uel_bert_data *data) {
+    uint8_t size;
+    uint8_t sign;
+    
+    size = uel_read8(data);
+    sign = uel_read8(data);
+
+    uel_check_bounds(data, size);
+    data->buff += size;
+
+    return rb_integer_unpack(data->buff - size, size, 1, 0, INTEGER_PACK_LITTLE_ENDIAN | (sign * INTEGER_PACK_NEGATIVE));
+}
+
+VALUE uel_read_large_bignum(struct uel_bert_data *data) {
+    uint32_t size;
+    uint8_t sign;
+
+    size = uel_read32(data);
+    sign = uel_read8(data);
+
+    uel_check_bounds(data, size);
+    data->buff += size;
+    return rb_integer_unpack(data->buff - size, size, 1, 0, INTEGER_PACK_LITTLE_ENDIAN | (sign * INTEGER_PACK_NEGATIVE));
+}
+
+VALUE uel_read_new_float(struct uel_bert_data *data) {
+    DOUBLE_SWAPPER tmp;
+
+    memcpy(tmp.buff, data->buff, sizeof(double));
+    data->buff += sizeof(double);
+    tmp.u = be64toh(tmp.u);
+
+    return DBL2NUM(tmp.d);
+}
+
+inline void uel_read_n_terms(struct uel_bert_data *data, VALUE *values, uint32_t count) {
+    for(uint32_t index = 0; index < count; index++) {
+        values[index] = uel_decode_term(data);
+    }
+}
+
+VALUE uel_read_list(struct uel_bert_data *data) {
+    uint32_t length;
+    VALUE* values;
+
+    length = uel_read32(data);
+    // Extra value to account for Tail
+    values = (VALUE *)malloc(sizeof(VALUE) * (length + 1));
+
+    uel_read_n_terms(data, values, length);
+
+    uel_check_bounds(data, 1);
+    if (*data->buff == UEL_ETF_NIL_ID) {
+        data->buff += 1;
+        return rb_ary_new_from_values(length, values);
+    }
+    else {
+        values[length] = uel_decode_term(data);
+        return rb_ary_new_from_values(length + 1, values);
+    }
+
+}
+
+VALUE uel_read_map(struct uel_bert_data *data) {
+    uint32_t arity;
+    VALUE hash;
+    VALUE key;
+    VALUE value;
+
+    arity = uel_read32(data);
+    hash = rb_hash_new();
+    for (uint32_t index = 0; index < arity; index++) {
+        key = uel_decode_term(data);
+        value = uel_decode_term(data);
+        rb_hash_aset(hash, key, value);
+    }
+
+    return hash;
+}
+
+VALUE uel_read_binary(struct uel_bert_data *data) {
+    uint32_t length;
+    
+    length = uel_read32(data);
+    uel_check_bounds(data, length);
+    data->buff += length;
+    return rb_str_new((const char *)data->buff - length, length);
+}
+
+VALUE uel_read_small_tuple(struct uel_bert_data *data) {
+    uint8_t length;
+    VALUE *values;
+
+    length = uel_read8(data);
+    values = (VALUE *)malloc(sizeof(VALUE) * length);
+
+    uel_read_n_terms(data, values, length);
+
+    return rb_ary_new_from_values(length, values);
+}
+
+VALUE uel_read_large_tuple(struct uel_bert_data *data) {
+    uint32_t length;
+    VALUE *values;
+
+    length = uel_read32(data);
+    values = (VALUE *)malloc(sizeof(VALUE) * length);
+
+    uel_read_n_terms(data, values, length);
+
+    return rb_ary_new_from_values(length, values);
+}
+
+VALUE uel_read_symbol(struct uel_bert_data *data, uint16_t length) {
+    VALUE string;
+
+    string = rb_enc_str_new((const char *)data->buff, length, rb_utf8_encoding());
+    data->buff += length;
+    return rb_to_symbol(string);
+}
+
+VALUE uel_read_atom(struct uel_bert_data *data) {
+    return uel_read_symbol(data, uel_read16(data));
+}
+
+VALUE uel_read_small_atom(struct uel_bert_data *data) {
+    return uel_read_symbol(data, uel_read8(data));
+}
+
+/* 
+    TODO: 
+        SMALL_TUPLE
+        LARGE_TUPLE
+        STRING
+        ATOM_SHIT
+*/
+
+#if HAVE_ZLIB_H
+VALUE uel_read_zlib(struct uel_bert_data *data) {
+    uint32_t inflate_length;
+    int inflate_status_code;
+    z_stream strm;
+    unsigned char *out;
+    struct uel_bert_data new_data;
+
+    inflate_length = uel_read32(data);
+    out = malloc(inflate_length * sizeof(uint8_t));
+
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = (uint32_t)UEL_DATA_LEN(data);
+    strm.next_in = data->buff;
+    strm.avail_out = inflate_length;
+    strm.next_out = out;
+
+    inflateInit(&strm);
+    inflate(&strm, Z_NO_FLUSH);
+    inflate_status_code = inflateEnd(&strm);
+
+    new_data.buff = out;
+    new_data.end = out + inflate_length;
+
+    if (inflate_status_code != Z_OK) {
+        rb_raise(rb_eTypeError, "Error inflating compressed data: (%i) %s", inflate_status_code, strm.msg);
+    }
+
+    return uel_decode_term(&new_data);
+}
+#endif

data/ext/uel/encode.c

@@ -0,0 +1,282 @@
+#include <netinet/in.h>
+#include <endian.h>
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <string.h>
+#include <stdlib.h>
+#include "./extconf.h"
+#include "./uel.h"
+
+VALUE uel_encode(VALUE self, VALUE value) {
+    uint8_t *buff;
+    VALUE ret_val;
+    struct uel_bert_data *term;
+    uint32_t term_len;
+
+    term = uel_encode_term(value);
+    if (term == NULL) {
+        rb_raise(rb_eStandardError, "Error decoding term");
+        return Qnil;
+    }
+
+
+    term_len = UEL_DATA_LEN(term);
+    buff = malloc(sizeof(uint8_t) * (term_len + 1));
+    buff[0] = 131;
+    memcpy(&buff[1], term->buff, term_len);
+    ret_val = rb_str_new((const char *)buff, (term_len + 1));
+    
+    free(buff);
+    uel_destroy_bert_data(term);
+
+    return ret_val;
+}
+
+
+struct uel_bert_data *uel_encode_term(VALUE value) {    
+    switch (TYPE(value)) {
+        case T_FIXNUM:
+            if (FIX2LONG(value) <= 0xFFFFFFFF)
+                return uel_encode_fixnum(value);
+            else
+                return uel_encode_bignum(value);
+        case T_FLOAT:
+            return uel_encode_float(value);
+        case T_BIGNUM:
+            return uel_encode_bignum(value);
+        case T_NIL:
+            return uel_encode_ptr_small_atom("nil", 3);
+        case T_TRUE:
+            return uel_encode_ptr_small_atom("true", 4);
+        case T_FALSE:
+            return uel_encode_ptr_small_atom("false", 5);
+        case T_SYMBOL:
+            return uel_encode_symbol(value);
+        case T_ARRAY:
+            return uel_encode_array(value);
+        case T_HASH:
+            return uel_encode_hash(value);
+        case T_STRING:
+            return uel_encode_string(value);
+        default:
+            /* TODO: Include class name
+                rb_raise(rb_eNotImpError, "Type not implemented %s", rb_class2name(RBASIC_CLASS(value)));
+            */
+            return NULL;
+    }
+}
+
+inline void uel_destroy_bert_data(struct uel_bert_data* data) {
+    free(data->buff);
+    free(data);
+}
+
+struct uel_bert_data *uel_encode_fixnum(VALUE fixnum) {
+    long l;
+    struct uel_bert_data *data = NULL;
+
+    l = FIX2LONG(fixnum);
+    if (l <= 0xFF) {
+        data = malloc(sizeof(struct uel_bert_data));
+        data->buff = malloc(sizeof(uint8_t) * 2);
+        data->buff[0] = UEL_ETF_SMALL_INTEGER_ID;
+        data->buff[1] = (uint8_t)l;
+        UEL_DATA_SET_LEN(data, 2);
+    } else {
+        data = malloc(sizeof(struct uel_bert_data));
+        data->buff = malloc(sizeof(uint8_t) * 5);
+        data->buff[0] = UEL_ETF_INTEGER_ID;
+
+        UEL_SET_INT32(data->buff + 1, l);
+        UEL_DATA_SET_LEN(data, 5);
+    }
+
+    return data;
+}
+
+struct uel_bert_data *uel_encode_float(VALUE value) {
+    struct uel_bert_data *data = malloc(sizeof(struct uel_bert_data));
+    DOUBLE_SWAPPER swapper;
+
+    data->buff = (uint8_t *)malloc(sizeof(uint8_t) * 9);
+    UEL_DATA_SET_LEN(data, 9);
+
+    swapper.d = NUM2DBL(value);
+    swapper.u = htobe64(swapper.u);
+
+    *data->buff = UEL_ETF_NEW_FLOAT_ID;
+    memcpy(data->buff + 1, swapper.buff, sizeof(double));
+
+    return data;
+}
+
+struct uel_bert_data *uel_encode_bignum(VALUE value) {
+    size_t byte_count;
+    struct uel_bert_data *data = NULL;
+
+    data = malloc(sizeof(struct uel_bert_data));
+    byte_count = rb_absint_size(value, NULL);
+    if (byte_count <= 0xFF) {
+        // id byte | n byte | sign byte | data
+        data->buff = malloc(sizeof(uint8_t) * (3 + byte_count));
+        data->buff[0] = UEL_ETF_SMALL_BIGNUM_ID;
+        data->buff[1] = (uint8_t)byte_count;
+        data->buff[2] = FIX2LONG(value) >= 0 ? 0 : 1;
+        UEL_DATA_SET_LEN(data, 3 + byte_count);
+
+        rb_integer_pack(value, data->buff + 3, byte_count, sizeof(uint8_t), 0, INTEGER_PACK_LITTLE_ENDIAN);
+    }
+    else {
+        // id byte | 4 byte n | sign byte | data
+        data->buff = malloc(sizeof(uint8_t) * (2 + byte_count) + sizeof(uint32_t));
+        data->buff[0] = UEL_ETF_LARGE_BIGNUM_ID;
+        UEL_SET_INT32(data->buff + 1, byte_count);
+        data->buff[5] = RBIGNUM_SIGN(value) ? 0 : 1;
+        UEL_DATA_SET_LEN(data, 6 + byte_count);
+
+        rb_integer_pack(value, data->buff + 6, byte_count, sizeof(uint8_t), 0, INTEGER_PACK_LITTLE_ENDIAN);
+    }
+    return data;
+}
+
+struct uel_bert_data *uel_encode_ptr_atom(const char *name, uint16_t len) {
+    struct uel_bert_data *data = malloc(sizeof(struct uel_bert_data));
+    
+    data->buff = malloc(sizeof(uint8_t) * (len + 3));
+    data->buff[0] = UEL_ETF_ATOM_UTF8_ID;
+    UEL_SET_UINT16(data->buff + 1, len);
+    memcpy(data->buff + 3, name, len);
+    UEL_DATA_SET_LEN(data, 3 + len);
+
+    return data;
+}
+
+struct uel_bert_data *uel_encode_ptr_small_atom(const char *name, uint8_t len) {
+    struct uel_bert_data *data = malloc(sizeof(struct uel_bert_data));
+
+    data->buff = malloc(sizeof(uint8_t) * (len + 2));
+    data->buff[0] = UEL_ETF_SMALL_ATOM_UTF8_ID;
+    data->buff[1] = len;
+    memcpy(data->buff + 2, name, len);
+    UEL_DATA_SET_LEN(data, 2 + len);
+
+    return data;
+}
+
+struct uel_bert_data *uel_encode_symbol(VALUE value) {
+    const char *name;
+    uint16_t len;
+
+    name = rb_id2name(rb_to_id(value));
+    len = strlen(name);
+
+    if (len > 0xFF)
+        return uel_encode_ptr_atom(name, len);
+    else
+        return uel_encode_ptr_small_atom(name, len);
+}
+
+struct uel_bert_data *uel_encode_array(VALUE value) {
+    uint32_t data_size = 0;
+    // Account for id tag and length
+    uint32_t offset = 5;
+    uint32_t elem_count = RARRAY_LEN(value);
+    struct uel_bert_data **data_ary = malloc(sizeof(struct uel_bert_data *) * elem_count);
+    struct uel_bert_data *element;
+    struct uel_bert_data *ret_data;
+
+    for (uint32_t i = 0; i < elem_count; i++) {
+        element = uel_encode_term(rb_ary_entry(value, i));
+        if (element == NULL) {
+            for (uint32_t j = 0; j <= i; j++)
+                uel_destroy_bert_data(data_ary[j]);
+            free(data_ary);
+            return NULL;
+        }
+        data_size += UEL_DATA_LEN(element);
+        data_ary[i] = element;
+    }
+
+    ret_data = malloc(sizeof(struct uel_bert_data));
+    ret_data->buff = malloc(sizeof(uint8_t) * (data_size + 6));
+    ret_data->buff[0] = UEL_ETF_LIST_ID;
+    UEL_SET_UINT32(ret_data->buff + 1, elem_count);
+    UEL_DATA_SET_LEN(ret_data, data_size + 6);
+
+    for (uint32_t i = 0; i < elem_count; i++) {
+        memcpy(ret_data->buff + offset, data_ary[i]->buff, UEL_DATA_LEN(data_ary[i]));
+        offset += UEL_DATA_LEN(data_ary[i]);
+        uel_destroy_bert_data(data_ary[i]);
+    }
+    ret_data->buff[offset] = UEL_ETF_NIL_ID;
+    free(data_ary);
+
+    return ret_data;
+}
+
+struct uel_bert_data *uel_encode_string(VALUE value) {
+    struct uel_bert_data *data;
+    size_t len = RSTRING_LEN(value);
+
+    data = malloc(sizeof(struct uel_bert_data));
+    data->buff = malloc(sizeof(uint8_t) * (len + 5));
+    UEL_DATA_SET_LEN(data, len + 5);
+
+    data->buff[0] = UEL_ETF_BINARY_ID;
+    UEL_SET_UINT32(data->buff + 1, len);
+    memcpy(data->buff + 5, StringValuePtr(value), len);
+
+    return data;
+}
+
+struct uel_bert_data *uel_encode_hash(VALUE value) {
+    struct uel_bert_data *data;
+    struct uel_bert_data **data_ary;
+    uint32_t hash_size;
+    int32_t data_index = 0;
+    uint32_t key_index = 0;
+    uint32_t data_size = 0;
+    uint32_t offset;
+    VALUE keys;
+    VALUE key;
+
+    hash_size = rb_hash_size_num(value);
+    data = malloc(sizeof(struct uel_bert_data));
+    data_ary = malloc(sizeof(struct uel_bert_data *) * (hash_size * 2));
+    keys = rb_hash_keys(value);
+
+    for (; key_index < hash_size; key_index++) {
+        key = rb_ary_entry(keys, key_index);
+        data_ary[data_index++] = uel_encode_term(key);
+        data_ary[data_index++] = uel_encode_term(rb_hash_lookup(value, key));
+
+        if (data_ary[data_index - 1] == NULL || data_ary[data_index - 2] == NULL) {
+            while(--data_index >= 0) {
+                if (data_ary[data_index] != NULL)
+                    uel_destroy_bert_data(data_ary[data_index]);
+            }
+            free(data);
+            free(data_ary);
+            return NULL;
+        }
+
+        data_size += UEL_DATA_LEN(data_ary[data_index - 1]);
+        data_size += UEL_DATA_LEN(data_ary[data_index - 2]);
+    }
+    
+    data->buff = (uint8_t *)malloc(sizeof(uint8_t) * (5 + data_size));
+    data->buff[0] = UEL_ETF_MAP_ID;
+    UEL_SET_UINT32(data->buff + 1, hash_size);
+    UEL_DATA_SET_LEN(data, 5 + data_size);
+    
+    offset = 0;
+    for (int index = 0; index < data_index; index++) {
+        memcpy(data->buff + 5 + offset, data_ary[index]->buff, UEL_DATA_LEN(data_ary[index]));
+        offset += UEL_DATA_LEN(data_ary[index]); 
+        uel_destroy_bert_data(data_ary[index]);
+    }
+
+    free(data_ary);
+
+    return data;
+}

data/ext/uel/extconf.rb

@@ -0,0 +1,14 @@
+require 'mkmf'
+
+extension_name = 'uel'
+dir_config(extension_name)
+find_header('ruby.h')
+find_header('endian.h')
+find_header('stdlib.h')
+
+if have_library('z')
+  have_header('zlib.h')
+end
+
+create_header
+create_makefile(extension_name)

data/ext/uel/uel.c

@@ -0,0 +1,19 @@
+#include <netinet/in.h>
+#include <endian.h>
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <string.h>
+#include <stdlib.h>
+#include "./extconf.h"
+#include "./uel.h"
+
+
+
+VALUE UELModule = Qnil;
+
+void Init_uel() {
+    UELModule = rb_define_module("UEL");
+    rb_define_module_function(UELModule, "decode", uel_decode, 1);
+    rb_define_module_function(UELModule, "encode", uel_encode, 1);
+}
+

data/ext/uel/uel.h

@@ -0,0 +1,110 @@
+#ifndef UNTITLED_ETF_LIB_H
+#define UNTITLED_ETF_LIB_H
+
+#define UEL_ETF_VERSION_NUMBER 131
+
+/* Supported Terms */
+#define UEL_ETF_NEW_FLOAT_ID 70
+#define UEL_ETF_BIT_BINARY_ID 77
+#define UEL_ETF_ZLIB_ID 80
+#define UEL_ETF_SMALL_INTEGER_ID 97
+#define UEL_ETF_INTEGER_ID 98
+#define UEL_ETF_FLOAT_ID 99
+#define UEL_ETF_ATOM_ID 100
+#define UEL_ETF_SMALL_TUPLE_ID 104
+#define UEL_ETF_LARGE_TUPLE_ID 105
+#define UEL_ETF_NIL_ID 106
+#define UEL_ETF_STRING_ID 107
+#define UEL_ETF_LIST_ID 108
+#define UEL_ETF_BINARY_ID 109
+#define UEL_ETF_SMALL_BIGNUM_ID 110
+#define UEL_ETF_LARGE_BIGNUM_ID 111
+#define UEL_ETF_SMALL_ATOM_ID 115
+#define UEL_ETF_MAP_ID 116
+#define UEL_ETF_ATOM_UTF8_ID 118
+#define UEL_ETF_SMALL_ATOM_UTF8_ID 119
+
+
+/* Unsupported terms */
+#define ETF_ATOM_CACHE_REF_ID 82
+#define ETF_NEW_PID_ID 88
+#define ETF_NEW_PORT_ID 89
+#define ETF_NEWER_REFERENCE_ID 90
+#define ETF_REFERENCE_ID 101
+#define ETF_PORT_ID 102
+#define ETF_PID_ID 103
+#define ETF_NEW_FUN_ID 112
+#define ETF_EXPORT_ID 113
+#define ETF_NEW_REFERENCE_ID 114
+#define ETF_FUN_ID 117
+
+
+struct uel_bert_data {
+    uint8_t *buff;
+    uint8_t *end;
+    uint32_t length;
+};
+
+#define UEL_DATA_LEN(etf_data) (etf_data->end - etf_data->buff)
+#define UEL_DATA_SET_LEN(etf_data, len) (etf_data->end = etf_data->buff + (len))
+#define UEL_SET_CAST(type, data, value) *((type *)(data)) = (type)value;
+#define UEL_SET_INT32(data, value) UEL_SET_CAST(int32_t, (data), htobe32(value))
+#define UEL_SET_INT16(data, value) UEL_SET_CAST(int16_t, (data), htobe16(value))
+#define UEL_SET_UINT32(data, value) UEL_SET_CAST(uint32_t, (data), htobe32(value))
+#define UEL_SET_UINT16(data, value) UEL_SET_CAST(uint16_t, (data), htobe16(value))
+
+
+typedef union {
+    double d;
+    uint64_t u;
+    char buff[8];
+} DOUBLE_SWAPPER;
+
+void Init_uel();
+
+void uel_destroy_bert_data(struct uel_bert_data*data);
+
+/* decode */
+VALUE uel_decode(VALUE self, VALUE data);
+VALUE uel_decode_term(struct uel_bert_data *data);
+void uel_check_bounds(struct uel_bert_data *data, size_t size);
+uint8_t uel_read8(struct uel_bert_data *data);
+uint16_t uel_read16(struct uel_bert_data *data);
+uint32_t uel_read32(struct uel_bert_data *data);
+VALUE uel_read_small_integer(struct uel_bert_data *data);
+VALUE uel_read_integer(struct uel_bert_data *data);
+VALUE uel_read_float(struct uel_bert_data *data);
+VALUE uel_read_nil(struct uel_bert_data *data);
+VALUE uel_read_small_bignum(struct uel_bert_data *data);
+VALUE uel_read_large_bignum(struct uel_bert_data *data);
+VALUE uel_read_new_float(struct uel_bert_data *data);
+VALUE uel_read_list(struct uel_bert_data *data);
+VALUE uel_read_map(struct uel_bert_data *data);
+VALUE uel_read_binary(struct uel_bert_data *data);
+VALUE uel_read_zlib(struct uel_bert_data *data);
+VALUE uel_read_small_tuple(struct uel_bert_data *data);
+VALUE uel_read_large_tuple(struct uel_bert_data *data);
+VALUE uel_read_atom(struct uel_bert_data *data);
+VALUE uel_read_small_atom(struct uel_bert_data *data);
+VALUE uel_read_atom_utf8(struct uel_bert_data *data);
+VALUE uel_read_small_atom_utf8(struct uel_bert_data *data);
+
+/* encode */
+VALUE uel_encode(VALUE self, VALUE value);
+struct uel_bert_data *uel_encode_term(VALUE value);
+struct uel_bert_data *uel_encode_fixnum(VALUE value);
+struct uel_bert_data *uel_encode_float(VALUE value);
+struct uel_bert_data *uel_encode_bignum(VALUE value);
+struct uel_bert_data *uel_encode_array(VALUE value);
+struct uel_bert_data *uel_encode_hash(VALUE value);
+struct uel_bert_data *uel_encode_struct(VALUE value);
+struct uel_bert_data *uel_encode_array(VALUE value);
+struct uel_bert_data *uel_encode_symbol(VALUE value);
+struct uel_bert_data *uel_encode_string(VALUE value);
+struct uel_bert_data *uel_encode_ptr_atom(const char *name, uint16_t len);
+struct uel_bert_data *uel_encode_ptr_small_atom(const char *name, uint8_t len);
+
+/* ruby symbols */
+VALUE rb_hash_keys(VALUE hash);
+
+#endif

data/uel.gemspec

@@ -0,0 +1,29 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "UEL"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Matthew Carey"]
+  spec.email         = ["matthew.b.carey@gmail.com"]
+
+  spec.summary       = %q{untitled etf lib}
+  spec.description   = %q{untitled etf lib}
+  spec.homepage      = "https://github.com/swarley/uel"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.extensions    << "ext/uel/extconf.rb"
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake-compiler"
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: UEL
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matthew Carey
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: untitled etf lib
+email:
+- matthew.b.carey@gmail.com
+executables: []
+extensions:
+- ext/uel/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark.rb
+- ext/uel/decode.c
+- ext/uel/encode.c
+- ext/uel/extconf.rb
+- ext/uel/uel.c
+- ext/uel/uel.h
+- uel.gemspec
+homepage: https://github.com/swarley/uel
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/swarley/uel
+  source_code_uri: https://github.com/swarley/uel
+  changelog_uri: https://github.com/swarley/uel
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: untitled etf lib
+test_files: []