divine 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in babelphish.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2012 Nils Franzen
+Copyright (c) 2009-2010, Patrick Reiter Horn (Javascript UTF8 string encode/decode methods taken from the ProtoJS project)
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,122 @@
+# Divine
+
+Divine provides a compact data interchange format for Ruby, Java and Javascript. Divine is similar to [Thrift](http://thrift.apache.org/) and [Protobuf](http://code.google.com/p/protobuf/), but I try to overcome of some of the shortcommings I think the other libraries have. 
+
+
+## Example
+
+
+```ruby
+require 'divine'
+
+struct 'TestBasic' do
+  int8 :i8
+  int16 :i16
+  int32 :i32
+  string :str
+  ip_number :ip
+  binary :guid
+end
+
+struct 'Entry' do
+  
+end
+
+
+struct(:TestComplex) {
+  list :list1, :int32
+  list :list2, :int8
+  map :map1, :int8, :int32
+  map(:map2, :string) { 
+    list 'Entry'
+  }
+}
+
+Divine::CodeGenerator.new.generate(:ruby, file: 'test_babel.rb', module: 'BabelTest', parent_class: "Object")
+Divine::CodeGenerator.new.generate(:javascript, file: 'test_babel.js')
+```
+
+The resulting _test_babel.rb_ contains the generated source code for the defined structs. Below is an example on how to use the generated code in ruby
+
+```ruby
+require_relative 'test_babel.rb'
+
+t1 = BabelTest::TestBasic.new
+t1.i8 = 0x0F
+t1.i16 = 0X1234
+t1.i32 = 0x567890AB
+t1.str = "abc\n123\tåäö´+'\""
+t1.ip = "192.168.0.1"
+t1.guid = (0..255).to_a
+
+p1 = BabelTest::TestComplex.new
+p1.list1 = [0, 1, 255, 0x7FFFFFFF, 0x7FFFFFFF+1, 0xFFFFFFFE, 0xFFFFFFFF]
+p1.list2 = [0,1, 15,16, 127,128, 254,255]
+p1.map1[0] = 10
+p1.map1[10] = 100
+p1.map2["Hello"] = [BabelTest::Entry.new, BabelTest::Entry.new]
+
+
+data1 = t1.serialize
+data2 = p1.serialize
+File.open("bin.babel", "w+b") do |f|
+  f.write(data1)
+  f.write(data2)
+end
+
+mem_buf = File.new('bin.babel').binmode
+t2 = BabelTest::TestBasic.new
+t2.deserialize mem_buf
+
+p2 = BabelTest::TestComplex.new
+p2.deserialize mem_buf
+```
+
+And a javascript example that uses the generated file _test_babel.js_
+
+```javascript
+var c1 = new TestComplex();
+c1.list1 = [0, 1, 255, 0x7FFFFFFF, 0x7FFFFFFF+1, 0xFFFFFFFE, 0xFFFFFFFF];
+c1.list2 = [0,1, 15,16, 127,128, 254,255];
+
+c1.map1[0] = 10;
+c1.map1[10] = 100;
+
+c1.map2["FooBar"] = [new Entry(), new Entry()];
+
+
+console.log("SERIALIZE");
+var ca = c1.serialize();
+
+console.log("DESERIALIZE");
+var c2 = new TestComplex();
+c2.deserialize(new BabelDataReader(ca));
+console.log(c2);
+```
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'divine'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install divine
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/divine.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'divine/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "divine"
+  gem.version       = Divine::VERSION
+  gem.authors       = ["Nils Franzen"]
+  gem.email         = ["nils@franzens.org"]
+  gem.description   = %q{A simple data serialization generator for java, ruby and javascript}
+  gem.summary       = %q{A simple data serialization generator for java, ruby and javascript}
+  gem.homepage      = "https://github.com/franzen/babelphish"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_dependency(%q<docile>, [">= 1.0.0"])
+  gem.add_dependency(%q<erubis>, [">= 2.7.0"])
+end

data/lib/divine.rb

@@ -0,0 +1,22 @@
+require 'erubis'
+require 'docile'
+
+require "divine/version"
+require "divine/dsl"
+require "divine/code_generators/code_generator"
+require "divine/code_generators/ruby"
+require "divine/code_generators/java"
+require "divine/code_generators/javascript"
+
+module Divine
+end
+
+#
+# Toplevel definition of struct'
+#
+def struct(name, version=1, &block)
+  puts "struct #{name}"
+  builder = Divine::StructBuilder.new(name, version) # Defined in divine/dsl.rb
+  ::Docile.dsl_eval(builder, &block)
+  builder
+end

data/lib/divine/code_generators/code_generator.rb

@@ -0,0 +1,49 @@
+module Divine
+  $language_generators = {}
+  
+  class BabelHelperMethods
+    def format_src(first_indent, following_indent, is, spc = " ")
+      indent = "#{spc * first_indent}"
+      is.flatten.compact.map do |i|
+        case i
+        when :indent
+          indent << spc * following_indent
+          nil
+        when :deindent
+          indent = indent[0..-(following_indent+1)]
+          nil
+        else
+          "#{indent}#{i}"
+        end
+      end.compact.join("\n")
+    end
+    
+    def get_fresh_variable_name
+      @vindex = (@vindex || 0xFF) + 1
+      return "var_#{@vindex.to_s(16)}"
+    end
+    
+    def camelize(*str)
+      ss = str.map(&:to_s).join("_").split(/_/).flatten
+      "#{ss.first.downcase}#{ss[1..-1].map(&:downcase).map(&:capitalize).join}"
+    end
+  end
+  
+  class CodeGenerator
+    def generate(target, opts)
+      gen = $language_generators[target.to_sym]
+      raise "Unknown taget language: #{target}" unless gen
+      puts "Generating code for #{target}"
+      src = gen.generate_code($all_structs, opts)
+      if opts[:file]
+        filename = opts[:file]
+        File.open(filename, 'w+') do |f|
+          puts "... writing #{filename}"
+          f.write(src)
+        end
+      else
+        puts src
+      end
+    end
+  end
+end

data/lib/divine/code_generators/java.rb

@@ -0,0 +1,414 @@
+module Divine
+
+  class JavaHelperMethods < BabelHelperMethods
+    def java_base_class_template_str
+      <<EOS
+abstract class BabelBase <%= toplevel_class %> {
+	private static final Charset UTF8 = Charset.forName("UTF-8");
+
+	public byte[] serialize() throws IOException {
+		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+		serializeInternal(baos);
+		baos.close();
+		return baos.toByteArray();
+	}
+
+	abstract void serializeInternal(ByteArrayOutputStream baos) throws IOException;
+
+	abstract void deserialize(ByteArrayInputStream baos) throws IOException;
+
+	protected int readInt8(ByteArrayInputStream data) {
+		return data.read() & 0xff;
+	}
+
+	protected int readInt16(ByteArrayInputStream data) {
+		return (data.read() << 8) | readInt8(data);
+	}
+
+	protected int readInt24(ByteArrayInputStream data) {
+		return (data.read() << 16) | readInt16(data);
+	}
+
+	protected long readInt32(ByteArrayInputStream data) {
+		return (data.read() << 24) | readInt24(data);
+	}
+
+	protected boolean readBool(ByteArrayInputStream data) {
+		return readInt8(data) == 1;
+	}
+
+	protected String readString(ByteArrayInputStream data) throws IOException {
+		// Force utf8
+		return new String(readBytes(readInt16(data), data), UTF8);
+	}
+
+	private byte[] readBytes(int size, ByteArrayInputStream data) throws IOException {
+		byte[] bs = new byte[size];
+		data.read(bs);
+		return bs;
+	}
+
+	protected byte[] readBinary(ByteArrayInputStream data) throws IOException {
+		long c = readInt32(data);
+		if (c > Integer.MAX_VALUE) {
+			throw new IndexOutOfBoundsException("Binary data to big for java");
+		}
+		return readBytes((int) readInt32(data), data);
+	}
+
+	protected byte[] readShortBinary(ByteArrayInputStream data) throws IOException {
+		return readBytes(readInt8(data), data);
+	}
+
+	protected String readIpNumber(ByteArrayInputStream data) throws IOException {
+		byte[] ips = readShortBinary(data);
+		String ip = "";
+		for (byte b : ips) {
+			if (ip.length() > 0) {
+				ip += ".";
+			}
+			ip += (b & 0xFF);
+		}
+		return ip;
+	}
+
+	protected void writeInt8(int v, ByteArrayOutputStream out) {
+		if (v > 0xFF) { // Max 255
+			raiseError("Too large int8 number: " + v);
+		}
+		out.write(v);
+	}
+
+	protected void writeInt16(int v, ByteArrayOutputStream out) {
+		if (v > 0xFFFF) { // Max 65.535 
+			raiseError("Too large int16 number: " + v);
+		}
+		writeInt8(v >> 8 & 0xFF, out);
+		writeInt8(v & 0xFF, out);
+	}
+
+	protected void writeInt24(int v, ByteArrayOutputStream out) {
+		if (v > 0xFFFFFF) { // Max 16.777.215
+			raiseError("Too large int24 number: " + v);
+		}
+		writeInt8(v >> 16 & 0xFF, out);
+		writeInt16(v & 0xFFFF, out);
+	}
+
+	protected void writeInt32(long v, ByteArrayOutputStream out) {
+		if (v > 0xFFFFFFFF) { // Max 4.294.967.295
+			raiseError("Too large int32 number: " + v);
+		}
+		writeInt8((int) ((v >> 24) & 0xFF), out);
+		writeInt24((int) (v & 0xFFFFFF), out);
+	}
+
+	protected void writeBool(boolean v, ByteArrayOutputStream out) {
+		writeInt8(v ? 1 : 0, out);
+	}
+
+	protected void writeString(String v, ByteArrayOutputStream out) throws IOException {
+		byte[] bs = v.getBytes(UTF8);
+		if (bs.length > 0xFFFF) {
+			raiseError("Too large string: " + bs.length + " bytes");
+		}
+		writeInt16(bs.length, out);
+		out.write(bs);
+	}
+
+	protected void writeBinary(byte[] v, ByteArrayOutputStream out) throws IOException {
+		if (v.length > 0xFFFFFFFF) {
+			raiseError("Too large binary: " + v.length + " bytes");
+		}
+		writeInt32(v.length, out);
+		out.write(v);
+	}
+
+	protected void writeShortBinary(byte[] v, ByteArrayOutputStream out) throws IOException {
+		if (v.length > 0xFF) {
+			raiseError("Too large short_binary: " + v.length + " bytes");
+		}
+		writeInt8(v.length, out);
+		out.write(v);
+	}
+
+	protected void writeIpNumber(String v, ByteArrayOutputStream out) throws IOException {
+		String[] bs = v.split(".");
+		byte[] ss = new byte[bs.length];
+		for (int i = 0; i < bs.length; i++) {
+			ss[i] = (byte) (Integer.parseInt(bs[i]) & 0xFF);
+		}
+		if (ss.length == 0 || ss.length == 4) {
+			writeShortBinary(ss, out);
+		} else {
+			raiseError("Unknown IP v4 number " + v); // Only IPv4 for now 
+		}
+	}
+
+	protected void raiseError(String msg) {
+		throw new IllegalArgumentException("[" + this.getClass().getCanonicalName() + "] " + msg);
+	}
+}
+EOS
+    end
+
+    def java_class_template_str
+      <<EOS2
+public class <%= c.name %> extends BabelBase {
+<% unless c.fields.empty? %>
+<% c.fields.each do |f| %>
+	public <%= this.java_get_type_declaration(f) %> <%= f.name %> = <%= this.java_get_empty_declaration(f) %>;
+<% end %>
+<% end %>
+
+	@Override
+	void serializeInternal(ByteArrayOutputStream baos) throws IOException {
+<% c.simple_fields.each do |f| %>
+		<%= this.camelize("write", f.type) %>(this.<%= f.name %>, baos);
+<% end %>
+<% c.complex_fields.each do |f| %>
+<%= this.java_serialize_complex f %>
+<% end %>
+	}
+
+	@Override
+	void deserialize(ByteArrayInputStream bais) throws IOException {
+<% c.simple_fields.each do |f| %>
+		this.<%= f.name %> = <%= this.camelize("read", f.type) %>(bais);
+<% end %>
+<% c.complex_fields.each do |f| %>
+<%= this.java_deserialize_complex f %>
+<% end %>
+	}
+}
+EOS2
+    end
+
+    def java_get_empty_declaration(types, is_reference_type = false)
+      if types.respond_to? :referenced_types
+        java_get_empty_declaration(types.referenced_types)
+
+      elsif types.respond_to? :first
+        case types.first
+        when :list
+          "new #{java_get_type_declaration(types, true)}()"
+        when :map
+          "new #{java_get_type_declaration(types, true)}()"
+        else
+          raise "Missing empty declaration for #{types}"
+        end
+
+      else
+        case types
+        when :binary, :short_binary
+          is_reference_type ? "new Byte[0]" : "new byte[0]"
+        when :int8, :int16
+          "0"
+        when :int32
+          "0L"
+        when :string, :ip_number
+          "\"\""
+        else
+          if $all_structs[types]
+            types
+          else
+            raise "Unkown field type #{types}"
+          end
+        end
+      end
+    end
+    
+    def java_get_type_declaration(types, is_reference_type = false)
+      if types.respond_to? :referenced_types
+        java_get_type_declaration(types.referenced_types, is_reference_type)
+
+      elsif types.respond_to? :first
+        case types.first
+        when :list
+          subtypes = java_get_type_declaration(types[1], true)
+          return "ArrayList<#{subtypes}>"
+        when :map
+          key_type = java_get_type_declaration(types[1], true)
+          value_type = java_get_type_declaration(types[2], true)
+          return "HashMap<#{key_type}, #{value_type}>"
+        else
+          raise "Missing serialization for #{types}"
+        end
+
+      else
+        case types
+        when :binary, :short_binary
+          is_reference_type ? "Byte[]" : "byte[]"
+        when :int8, :int16
+          is_reference_type ? "Integer" : "int"
+        when :int32
+          is_reference_type ? "Long" : "long"
+        when :string, :ip_number
+          "String"
+        else
+          if $all_structs[types]
+            types
+          else
+            raise "Unkown field type #{types}"
+          end
+        end
+      end
+    end
+    
+    def java_serialize_complex(field)
+      types = field.referenced_types
+      as = [
+            "// Serialize #{field.type} '#{field.name}'",
+            java_serialize_internal(field.name, types)
+           ]
+      format_src(2, 1, as, "\t")
+    end
+
+    def java_serialize_internal(var, types)
+      if types.respond_to? :first
+        case types.first
+        when :list
+          nv = get_fresh_variable_name
+          idx = get_fresh_variable_name
+          return [
+                  "writeInt32(#{var}.size(), baos);",
+                  "for(int #{idx}=0; #{idx}<#{var}.size(); #{idx}++) {",
+                  :indent,
+                  "#{java_get_type_declaration types[1]} #{nv} = #{var}.get(#{idx});",
+                  java_serialize_internal(nv, types[1]),
+                  :deindent,
+                  "}"
+                 ]
+        when :map
+          nv1 = get_fresh_variable_name      
+          nv2 = get_fresh_variable_name
+          return [
+                  "writeInt32(#{var}.size(), baos);",
+                  "for(#{java_get_type_declaration types[1]} #{nv1} : #{var}.keySet()) {",
+                  :indent,
+                  "#{java_get_type_declaration types[2]} #{nv2} = #{var}.get(#{nv1});",
+                  java_serialize_internal(nv1, types[1]),
+                  java_serialize_internal(nv2, types[2]),
+                  :deindent,
+                  "}"
+                 ]
+        else
+          raise "Missing serialization for #{var}"
+        end
+      else
+        if $all_structs[types]
+          "#{var}.serializeInternal(baos);"
+      
+        elsif $available_types[types] && $available_types[types].ancestors.include?(SimpleDefinition)
+          "#{self.camelize "write", types}(#{var}, baos);"
+      
+        else
+          raise "Missing code generation case #{types}"
+        end
+      end
+    end
+
+    def java_deserialize_complex(field)
+      types = field.referenced_types
+      as = [
+            "// Deserialize #{field.type} '#{field.name}'",
+            java_deserialize_internal("this.#{field.name}", types)
+           ]
+      format_src(2, 1, as, "\t")
+    end
+
+    def java_deserialize_internal(var, types)
+      if types.respond_to? :first
+        case types.first
+        when :list
+          count = get_fresh_variable_name
+          nv = get_fresh_variable_name
+          iter = get_fresh_variable_name
+          return [
+                  "#{"#{java_get_type_declaration(types)} " unless var.include? "this."}#{var} = #{java_get_empty_declaration(types)};",
+                  "int #{count} = (int)this.readInt32(bais);",
+                  "for(int #{iter}=0; #{iter}<#{count}; #{iter}++) {",
+                  :indent,
+                  java_deserialize_internal(nv, types[1]),
+                  "#{var}.add(#{nv});",
+                  :deindent,
+                  "}"
+                 ]
+        when :map
+          count = get_fresh_variable_name
+          nv1 = get_fresh_variable_name      
+          nv2 = get_fresh_variable_name
+          iter = get_fresh_variable_name
+          return ["#{"#{java_get_type_declaration(types)} " unless var.include? "this."}#{var} = #{java_get_empty_declaration(types)};",
+                  "int #{count} = (int)readInt32(bais);",
+                  "for(int #{iter}=0; #{iter}<#{count}; #{iter}++) {",
+                  :indent,
+                  java_deserialize_internal(nv1, types[1]),
+                  java_deserialize_internal(nv2, types[2]),
+                  "#{var}.put(#{nv1}, #{nv2});",
+                  :deindent,
+                  "}"
+                 ]
+        else
+          raise "Missing serialization for #{var}"
+        end
+      else
+#        case types
+#        when :map
+#          "#{var} = #{java_get_empty_declaration(types)}"
+#        when :list
+#          "#{var} = #{java_get_empty_declaration(types)}"
+#        else
+          if $all_structs.key? types
+            [
+             "#{types} #{var} = new #{types}();",
+             "#{var}.deserialize(bais);"
+            ]
+          else
+            "#{"#{java_get_type_declaration(types)} " unless var.include? "this."}#{var} = #{self.camelize("read", types)}(bais);"
+          end
+#        end
+      end
+    end
+  end
+
+
+  class JavaGenerator < JavaHelperMethods
+    def generate_code(structs, opts)
+      pp opts
+      $debug_java = true if opts[:debug]
+      base_template = Erubis::Eruby.new(java_base_class_template_str)
+      class_template = Erubis::Eruby.new(java_class_template_str)
+      keys = structs.keys.sort
+      src = keys.map do |k|
+        ss = structs[k]
+        # TODO: Should we merge different versions and deduce deprecated methods, warn for incompatible changes, etc?
+        raise "Duplicate definitions of struct #{k}" if ss.size > 1
+        class_template.result( c: ss.first, this: self )
+      end
+    
+      # User defined super class?
+      toplevel = opts[:parent_class] || nil
+      toplevel = " extends #{toplevel}" if toplevel 
+      return "#{java_get_begin_module(opts)}#{base_template.result({ toplevel_class: toplevel })}\n\n#{src.join("\n\n")}"
+    end
+  
+    def java_get_begin_module(opts)
+      if opts[:package]
+        [
+          "package #{opts[:package]};\n",
+          "import java.io.ByteArrayInputStream;",
+          "import java.io.ByteArrayOutputStream;",
+          "import java.io.IOException;",
+          "import java.util.ArrayList;",
+          "import java.util.HashMap;",
+          "import java.nio.charset.Charset;\n\n"
+        ].join("\n")
+      else 
+        nil
+      end
+    end
+  end
+
+  $language_generators[:java] = JavaGenerator.new
+end