divine 0.0.2 → 0.0.3

data/.gitignore

@@ -15,3 +15,33 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.class
+test/basic_complex_test/java_test/bin.babel
+test/basic_complex_test/js_test/bin.babel.js
+test/basic_complex_test/ruby_test/bin.babel.rb
+test/binaryTree_test/java_test/bin.babel
+test/binaryTree_test/js_test/bin.babel.js
+test/binaryTree_test/ruby_test/bin.babel.rb
+test/complex_test/java_test/bin.babel
+test/complex_test/js_test/bin.babel.js
+test/complex_test/ruby_test/bin.babel.rb
+test/ipv6_test/java_test/bin.babel
+test/ipv6_test/js_test/bin.babel.js
+test/ipv6_test/ruby_test/bin.babel.rb
+test/basic_complex_test/ruby_test/test_babel.rb
+test/binaryTree_test/ruby_test/test_binaryTree.rb
+test/complex_test/ruby_test/test_complex.rb
+test/ipv6_test/ruby_test/test_ipv6.rb
+test/signed_int_test/ruby_test/test_signed_int.rb
+test/signed_int_test/js_test/bin.babel.js
+test/signed_int_test/ruby_test/bin.babel.rb
+test/basic_complex_test/java_test/test_babel.java
+test/binaryTree_test/java_test/test_binaryTree.java
+test/complex_test/java_test/test_complex.java
+test/ipv6_test/java_test/test_ipv6.java
+test/signed_int_test/java_test/test_signed_int.java
+test/basic_complex_test/js_test/test_babel.js
+test/binaryTree_test/js_test/test_binaryTree.js
+test/complex_test/js_test/test_complex.js
+test/ipv6_test/js_test/test_ipv6.js
+test/signed_int_test/js_test/test_signed_int.js

data/README.md

@@ -94,6 +94,63 @@ c2.deserialize(new BabelDataReader(ca));
 console.log(c2);
 ```
 
+### Versioning
+```ruby
+require 'divine'
+
+struct 'Foobar' do # Default version is '1'
+  int8 :foo
+end
+
+
+struct 'Foobar', version: 2 do
+  int8 :foo
+  int8 :bar # We added a new field named 'bar' in version 2 of Foobar
+end
+
+Divine::CodeGenerator.new.generate(:ruby, file: 'test_babel.rb')
+Divine::CodeGenerator.new.generate(:java, file: 'test_babel.java')
+```
+
+There are some basic rules regarding versioning of structs
+
+* A versioned struct defines all fields you want to serialize/deserialize
+* You can delete and add fields as you whish between versions
+* You are not allowed to change type of a defined variable between versions
+* You cannot have a bigger version number than 255
+* The class that represents the struct also defines a 'struct_version' that keeps the current version of the struct
+
+
+### Freezing
+When starting to use generated code in production, the defined structs cannot be changed without breaking everything (unless you really know what you're doing).
+To prevent that you change a struct by accident, you can 'freeze' a struct. The easiest way to get started is to add a empty _freeze_ field to the struct, like below
+
+```ruby
+require 'divine'
+
+struct 'Foobar', freeze: '' do
+  int8 :foo
+  int8 :bar
+end
+
+Divine::CodeGenerator.new.generate(:ruby, file: 'test_babel.rb')
+```
+
+The compiler will throw a runtime exception saying that the MD5 sum differs. Take the MD5 sum from this exception and put into the _freeze_ field as below 
+
+```ruby
+require 'divine'
+
+struct 'Foobar', freeze: '3e59aa582d3137a2d0cdba174e5aa6b18beb649b' do
+  int8 :foo
+  int8 :bar
+end
+
+Divine::CodeGenerator.new.generate(:ruby, file: 'test_babel.rb')
+```
+
+It is now not possible to alter _Foobar_ by accident. If you make a change to the struct, you will also need to provide a correct MD5 sum.
+
 
 ## Installation
 

data/Rakefile

@@ -1,27 +1,48 @@
 require "bundler/gem_tasks"
 
+
+task :default => ["test:all"]
+
+
 namespace :test  do
+  desc "Run all tests"
+  task :all => [:ruby, :js, :java, :unify]  	# Make sure that the unify test be the last one in the list.
   
   desc "Run Ruby code test suite"
-  task :ruby => :environment do
-    ruby "test/basic_complex_test/ruby_test/ruby_test.rb"
-    ruby "test/binaryTree_test/ruby_test/ruby_test.rb"
+  task :ruby do
+    generate_source('ruby')
+    ruby "test/signed_int_test/ruby_test/ruby_test.rb"
     ruby "test/ipv6_test/ruby_test/ruby_test.rb"
     ruby "test/complex_test/ruby_test/ruby_test.rb"
+    ruby "test/binaryTree_test/ruby_test/ruby_test.rb"
+    ruby "test/basic_complex_test/ruby_test/ruby_test.rb"
+
+    system("find . -name 'test*.rb' | xargs rm") # Remove generated source code files
   end
 
 
   desc "Run JS code test suite"
-  task :js => :environment do
-    system("node test/basic_complex_test/js_test/js_testBasic.js")
-    system("node test/basic_complex_test/js_test/js_testComplex.js")
-    system("node test/binaryTree_test/js_test/js_test.js")
+  task :js do
+    generate_source('js')
+    system("node test/signed_int_test/js_test/js_test.js")
     system("node test/ipv6_test/js_test/js_test.js")
     system("node test/complex_test/js_test/js_test.js")
+    system("node test/binaryTree_test/js_test/js_test.js")
+    system("node test/basic_complex_test/js_test/js_testBasic.js")
+    system("node test/basic_complex_test/js_test/js_testComplex.js")
+    
+    system("find . -name 'test*.js' | xargs rm") # Remove generated source code files
   end
 
   desc "Run java code test suite"
-  task :java => :environment do
+  task :java do
+    generate_source('java')
+    #system("javac -cp test/java_lib/junit.jar: test/signed_float_test/java_test/*.java")
+    #system("java -cp test/signed_float_test/java_test/:test/java_lib/junit.jar:. org.junit.runner.JUnitCore JavaTest")
+
+    system("javac -cp test/java_lib/junit.jar: test/signed_int_test/java_test/*.java")
+    system("java -cp test/signed_int_test/java_test/:test/java_lib/junit.jar:. org.junit.runner.JUnitCore JavaTest")
+    
     system("javac -cp test/java_lib/junit.jar: test/ipv6_test/java_test/*.java")
     system("java -cp test/ipv6_test/java_test/:test/java_lib/junit.jar:. org.junit.runner.JUnitCore JavaTest")
 
@@ -33,18 +54,24 @@ namespace :test  do
 
     system("javac -cp test/java_lib/junit.jar: test/basic_complex_test/java_test/*.java")
     system("java -cp test/basic_complex_test/java_test/:test/java_lib/junit.jar:. org.junit.runner.JUnitCore JavaTest")
-  end
-
 
-  desc "Generate Source Code"
-  task :environment do
-    puts "Divine Version #{Divine::VERSION}"
-    lang = Rake.application.top_level_tasks[0].match(/:(.*)/)[1]
+    system("find . -name 'test*.java' | xargs rm") # Remove generated source code files
+    system("find . -name '*.class' | xargs rm") # Remove generated .class files
+  end
 
-    ruby "test/basic_complex_test/basic_complex_test.rb #{lang}"
-    ruby "test/ipv6_test/ipv6_test.rb #{lang}"
-    ruby "test/complex_test/complex_test.rb #{lang}"
-    ruby "test/binaryTree_test/binaryTree_test.rb #{lang}"
+  desc "Unify test to compare the produced binary files. Prerequisite Other tests must be run first to generate bin files to be compared"
+  task :unify do
+     ruby "test/unify_test/unify_test.rb"
+     system("find . -name 'bin.babel*' | xargs rm") # Remove generated bin files
   end
 end
 
+
+def generate_source(lang)
+  puts "Divine Version #{Divine::VERSION}"
+  ruby "test/signed_int_test/signed_int_test.rb #{lang}"
+  ruby "test/ipv6_test/ipv6_test.rb #{lang}"
+  ruby "test/complex_test/complex_test.rb #{lang}"
+  ruby "test/binaryTree_test/binaryTree_test.rb #{lang}"
+  ruby "test/basic_complex_test/basic_complex_test.rb #{lang}"
+end

data/lib/divine.rb

@@ -14,9 +14,9 @@ end
 #
 # Toplevel definition of struct'
 #
-def struct(name, version=1, &block)
-  puts "struct #{name}"
-  builder = Divine::StructBuilder.new(name, version) # Defined in divine/dsl.rb
+def struct(name, properties=nil, &block)
+  #puts "struct #{name}"
+  builder = Divine::StructBuilder.new(name, properties) # Defined in divine/dsl.rb
   ::Docile.dsl_eval(builder, &block)
   builder
 end

data/lib/divine/code_generators/code_generator.rb

@@ -1,6 +1,29 @@
+require 'digest/sha1'
+require 'fileutils'
+
+
 module Divine
   $language_generators = {}
   
+  class StructHandler
+    attr_reader :name, :latest_version, :structs
+    
+    def initialize(structs)
+      @structs = structs
+      @name = structs.first.name
+      @latest_version = structs.last.version
+      @field_hash = structs.map(&:fields).flatten.group_by(&:name)
+    end
+    
+    def field(name)
+      @field_hash[name]
+    end
+    
+    def field_names
+      @field_hash.keys.sort
+    end
+  end
+  
   class BabelHelperMethods
     def format_src(first_indent, following_indent, is, spc = " ")
       indent = "#{spc * first_indent}"
@@ -27,40 +50,115 @@ module Divine
       ss = str.map(&:to_s).join("_").split(/_/).flatten
       "#{ss.first.downcase}#{ss[1..-1].map(&:downcase).map(&:capitalize).join}"
     end
+    
+    def get_header_comment_text
+      return [
+        "",
+        "-- DO NOT EDIT THIS FILE --", "",
+        "This file was generated by Divine #{Divine::VERSION} (#{Time.now.to_s})", "",
+        "-- DO NOT EDIT THIS FILE --", ""
+      ]
+    end
+    
+    #
+    # Sanity check the different revisions of the struct
+    #
+    def sanity_check(ss)
+      vos = ss.sort do |x,y|
+        v1,v2 = [x,y].map(&:version)
+        v1 <=> v2
+      end
+
+      # Check version numbers
+      unless vos.map(&:version).uniq.size == ss.size
+        raise "Inconsistent version numbering for '#{ss.first.name}': #{vos.map(&:version).join(', ')}"
+      end
+      
+      # Check that we don't have multiple variables with same name
+      vos.each do |s|
+        names = s.fields.group_by(&:name)
+        names.each_pair do |k, v|
+          raise "Multiple fields with same name '#{k}' is defined in '#{s.name}', version #{s.version}" unless v.size == 1
+        end
+      end
+      
+      # Check types between versions
+      check_field_types(vos)
+      
+      # Check for changed definitions
+      check_freezed_structs(vos)
+      
+      # Return them ordered by version
+      vos
+    end
+    
+    private
+    #
+    # Check types through all versions, we're not allowed to change the type of a variable between different versions
+    #
+    def check_field_types(ss)
+      types = {}
+      ss.each do |x|
+        x.fields.each do |f|
+          if types[f.name]
+            unless same_type(types[f.name], x, f.name)
+              raise "Cannot change the field type for struct #{x.name}.#{f.name} between version #{types[f.name].version} and #{x.version}"
+            end
+          else
+            types[f.name] = x
+          end
+        end
+      end
+    end
+    
+    def same_type(t1, t2, field)
+      t1.get_field(field).referenced_types == t2.get_field(field).referenced_types
+    end
+    
+    def check_freezed_structs(ss)
+      ss.each do |s|
+        if s.freezed?
+          sig = calculate_signature(s)
+          unless sig == s.freeze_signature
+            raise "Struct '#{s.name}', version #{s.version} has changed! Got signature '#{sig}', expected '#{s.freeze_signature}'"
+          end
+        end
+      end
+    end
+    
+    def calculate_signature(s)
+      str = s.fields.map do |f|
+        "#{f.name}:#{f.referenced_types.inspect}"
+      end.join(",")
+      Digest::SHA1.hexdigest("#{s.name}:#{s.version},#{str}")
+    end
   end
   
+  
+  
   class CodeGenerator
     def generate(target, opts)
       gen = $language_generators[target.to_sym]
-      raise "Unknown taget language: #{target}" unless gen
+      raise "Unknown target language: #{target}" unless gen
       puts "Generating code for #{target}"
       src = gen.generate_code($all_structs, opts)
-      target_dir = getTargetDir(opts[:target_dir])
+      target_dir = (opts[:target_dir] || ".") + "/"
 
       if opts[:package]
-        path = target_dir + opts[:package]
-        Dir.mkdir(path) unless File.exists?(path)
+        path = target_dir + opts[:package].gsub(/\./, "/")
+        FileUtils.mkdir_p(path) unless File.exists?(path)
         for cls in src
            file_name = path+"/"+cls[:file]
            writeFile(file_name, cls[:src])
         end
       elsif opts[:file]
+        FileUtils.mkdir_p(target_dir) unless File.exists?(target_dir)
         path = target_dir + opts[:file]
         writeFile(path, src[0][:src])
       else
         puts src
       end
     end
-
-    def getTargetDir(dir)
-      # check if the path is relative or absolute if exist
-      if dir && File.directory?(dir)
-        return dir + "/"
-      elsif dir && File.directory?(Dir.pwd + "/" + dir)
-        return Dir.pwd + dir + "/"
-      end
-      ""
-    end
     
     def writeFile(path, content)
       File.open(path, 'w+') do |f|

data/lib/divine/code_generators/java.rb

@@ -1,6 +1,12 @@
 module Divine
 
   class JavaHelperMethods < BabelHelperMethods
+    def get_header_comment
+      get_header_comment_text.map do |s|
+        "// #{s}"
+      end.join("\n")
+    end
+    
     def java_base_class_template_str
       <<EOS
 abstract class BabelBase <%= toplevel_class %> {
@@ -33,6 +39,14 @@ abstract class BabelBase <%= toplevel_class %> {
 		return (data.read() << 24) | readInt24(data);
 	}
 
+	protected int readSint32(ByteArrayInputStream data) {
+		return (data.read() << 24) | readInt24(data);
+	}
+
+	protected long readSint64(ByteArrayInputStream data) {
+		return (readInt32(data) << 32) | (readInt32(data) & 0xFFFFFFFFL);
+	}
+
 	protected boolean readBool(ByteArrayInputStream data) {
 		return readInt8(data) == 1;
 	}
@@ -135,6 +149,26 @@ abstract class BabelBase <%= toplevel_class %> {
 		writeInt24((int) (v & 0xFFFFFF), out);
 	}
 
+        protected void writeSint32(int v, ByteArrayOutputStream out) {
+		if (v > Integer.MAX_VALUE) { 		// Max  2.147.483.647
+			raiseError("Too large sInt32 number: " + v + ", Max = " + Integer.MAX_VALUE);
+		}else if(v < Integer.MIN_VALUE){ 	// Min -2.147.483.648
+			raiseError("Too small sInt32 number: " + v + ", Min = " + Integer.MIN_VALUE);
+		}
+		writeInt8((int) ((v >> 24) & 0xFF), out);
+		writeInt24((int) (v & 0xFFFFFF), out);
+	}
+
+	protected void writeSint64(long v, ByteArrayOutputStream out) {
+		if (v > Long.MAX_VALUE) { 		// Max  9,223,372,036,854,775,807
+			raiseError("Too large sInt64 number: " + v + ", Max = " + Integer.MAX_VALUE);
+		}else if(v < Long.MIN_VALUE){ 		// Min -9,223,372,036,854,775,807
+			raiseError("Too small sInt64 number: " + v + ", Min = " + Integer.MIN_VALUE);
+		}
+		writeInt32(((v >> 32) & 0xFFFFFFFFL), out);
+		writeInt32( (v & 0xFFFFFFFFL) , out);
+	}
+
 	protected void writeBool(boolean v, ByteArrayOutputStream out) {
 		writeInt8(v ? 1 : 0, out);
 	}
@@ -242,43 +276,90 @@ abstract class BabelBase <%= toplevel_class %> {
 EOS
     end
 
-    def java_class_template_str
-      <<EOS2
-class <%= c.name %> extends BabelBase {
-<% unless c.fields.empty? %>
-<% c.fields.each do |f| %>
-	public <%= this.java_get_type_declaration(f) %> <%= f.name %> = <%= this.java_get_empty_declaration(f) %>;
-<% end %>
-<% end %>
-
-	@Override
-	void serializeInternal(ByteArrayOutputStream baos) throws IOException {
-<% c.simple_fields.each do |f| %>
-		<%= this.camelize("write", f.type) %>(this.<%= f.name %>, baos);
-<% end %>
-<% c.complex_fields.each do |f| %>
-<%= this.java_serialize_complex f %>
-<% end %>
-	}
+    def java_class_template(sh)
+      code = [
+        "class #{sh.name} extends BabelBase {",
+        :indent,
+        "",
+
+        # PROPERTIES
+      	"public int struct_version = #{sh.latest_version};",
+        sh.field_names.map do |fn|
+          f = sh.field(fn).last
+        	"public #{java_get_type_declaration(f)} #{fn} = #{java_get_empty_declaration(f)};"
+        end, "",
+        
 
-	@Override
-	public void deserialize(ByteArrayInputStream bais) throws IOException {
-<% c.simple_fields.each do |f| %>
-		this.<%= f.name %> = <%= this.camelize("read", f.type) %>(bais);
-<% end %>
-<% c.complex_fields.each do |f| %>
-<%= this.java_deserialize_complex f %>
-<% end %>
-	}
-}
-EOS2
+        # SERiALIZE INTERNAL
+        "@Override",
+        "void serializeInternal(ByteArrayOutputStream baos) throws IOException {",
+        :indent,
+        "writeInt8(this.struct_version, baos);",
+        sh.structs.map do |s|
+          [
+            "if(this.struct_version == #{s.version}) {",
+            :indent,
+            s.simple_fields.map do |f|
+              "#{camelize("write", f.type)}(this.#{f.name}, baos);"
+            end,
+            s.complex_fields.map do |f|
+              [
+                "", "// Serialize #{f.type} '#{f.name}'",
+                java_serialize_internal("this.#{f.name}",  f.referenced_types)
+              ]
+            end,
+            "return;",
+            :deindent,
+            "}", ""
+          ]
+        end, "",
+        "throw new UnsupportedOperationException(\"Unsupported version \" + this.struct_version + \" for type '#{sh.name}'\");",
+        :deindent,
+        "}", "",
+
+
+        # DESERIALIZE
+        "@Override",
+        "public void deserialize(ByteArrayInputStream bais) throws IOException {",
+        :indent,
+        "this.struct_version = readInt8(bais);",
+        sh.structs.map do |s|
+          [
+            "if(this.struct_version == #{s.version}) {",
+            :indent,
+            s.simple_fields.map do |f|
+              "this.#{f.name} = #{camelize("read", f.type)}(bais);"
+            end,
+            s.complex_fields.map do |f|
+              [
+                "", "// Read #{f.type} '#{f.name}'",
+                java_deserialize_internal("this.#{f.name}",  f.referenced_types)
+              ]
+            end,
+            "return;",
+            :deindent,
+            "}"
+          ]
+        end, "",
+        "throw new UnsupportedOperationException(\"Unsupported version \" + this.struct_version + \" for type '#{sh.name}'\");",
+        :deindent,
+        "}", "",
+
+
+        # END OF CLASS
+        :deindent,
+        "}"
+      ]
+        
+      format_src(3, 3, code)
     end
+    
 
     def java_get_empty_declaration(types, is_reference_type = false)
       if types.respond_to? :referenced_types
         java_get_empty_declaration(types.referenced_types)
 
-      elsif types.respond_to? :first
+      elsif types.respond_to?(:first) && types.size > 1
         case types.first
         when :list
           "new #{java_get_type_declaration(types, true)}()"
@@ -288,13 +369,16 @@ EOS2
           raise "Missing empty declaration for #{types}"
         end
 
+      elsif types.respond_to?(:first) && types.size == 1
+        java_get_empty_declaration(types.first, is_reference_type)
+
       else
         case types
         when :binary, :short_binary
           is_reference_type ? "new Byte[0]" : "new byte[0]"
-        when :int8, :int16
+        when :int8, :int16, :sint32
           "0"
-        when :int32
+        when :int32, :sint64
           "0L"
         when :string, :ip_number
           "\"\""
@@ -312,7 +396,7 @@ EOS2
       if types.respond_to? :referenced_types
         java_get_type_declaration(types.referenced_types, is_reference_type)
 
-      elsif types.respond_to? :first
+      elsif types.respond_to?(:first) && types.size > 1
         case types.first
         when :list
           subtypes = java_get_type_declaration(types[1], true)
@@ -324,14 +408,17 @@ EOS2
         else
           raise "Missing serialization for #{types}"
         end
+        
+      elsif types.respond_to?(:first) && types.size == 1
+        java_get_type_declaration(types.first, is_reference_type)
 
       else
         case types
         when :binary, :short_binary
           is_reference_type ? "Byte[]" : "byte[]"
-        when :int8, :int16
+        when :int8, :int16, :sint32
           is_reference_type ? "Integer" : "int"
-        when :int32
+        when :int32, :sint64
           is_reference_type ? "Long" : "long"
         when :string, :ip_number
           "String"
@@ -345,15 +432,6 @@ EOS2
       end
     end
     
-    def java_serialize_complex(field)
-      types = field.referenced_types
-      as = [
-            "// Serialize #{field.type} '#{field.name}'",
-            java_serialize_internal(field.name, types)
-           ]
-      format_src(2, 1, as, "\t")
-    end
-
     def java_serialize_internal(var, types)
       if types.respond_to? :first
         case types.first
@@ -398,15 +476,6 @@ EOS2
       end
     end
 
-    def java_deserialize_complex(field)
-      types = field.referenced_types
-      as = [
-            "// Deserialize #{field.type} '#{field.name}'",
-            java_deserialize_internal("this.#{field.name}", types)
-           ]
-      format_src(2, 1, as, "\t")
-    end
-
     def java_deserialize_internal(var, types)
       if types.respond_to? :first
         case types.first
@@ -465,16 +534,14 @@ EOS2
 
   class JavaGenerator < JavaHelperMethods
     def generate_code(structs, opts)
-      pp opts
       $debug_java = true if opts[:debug]
       base_template = Erubis::Eruby.new(java_base_class_template_str)
-      class_template = Erubis::Eruby.new(java_class_template_str)
       keys = structs.keys.sort
       src = keys.map do |k|
         ss = structs[k]
-        # TODO: Should we merge different versions and deduce deprecated methods, warn for incompatible changes, etc?
-        raise "Duplicate definitions of struct #{k}" if ss.size > 1
-        class_template.result( c: ss.first, this: self )
+        # Check different aspects the the structs
+        vss = sanity_check(ss)
+        java_class_template(StructHandler.new(vss))
       end
     
       # User defined super class?
@@ -492,28 +559,25 @@ EOS2
     end
   
     def java_get_begin_module(opts)
-      if opts[:package]
-        [
-          "package #{opts[:package]};\n",
-          "import java.io.ByteArrayInputStream;",
-          "import java.io.ByteArrayOutputStream;",
-          "import java.io.IOException;",
-          "import java.util.ArrayList;",
-          "import java.util.HashMap;",
-          "import java.util.regex.Pattern;",
-          "import java.nio.charset.Charset;\n\n"
-        ].join("\n")
-      else 
-        [
-          "import java.io.ByteArrayInputStream;",
-          "import java.io.ByteArrayOutputStream;",
-          "import java.io.IOException;",
-          "import java.util.ArrayList;",
-          "import java.util.HashMap;",
-          "import java.util.regex.Pattern;",
-          "import java.nio.charset.Charset;\n\n"
-        ].join("\n")
-      end
+      str = "#{get_header_comment}\n\n"
+      str << "package #{opts[:package]};" if opts[:package]
+      str << get_java_imports
+      str << "\n\n"
+      return str
+    end
+    
+    def get_java_imports
+      [
+        "java.io.ByteArrayInputStream",
+        "java.io.ByteArrayOutputStream",
+        "java.io.IOException",
+        "java.util.ArrayList",
+        "java.util.HashMap",
+        "java.util.regex.Pattern",
+        "java.nio.charset.Charset"
+        ].map do |i|
+          "import #{i};"
+        end.join("\n") 
     end
   end