sereal 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8247824a3c96a8a8a0ef7a190fed6d8517ba85dc
-  data.tar.gz: 6e8edec6c3143ecd94fb22d6c96bb33664fd706b
+  metadata.gz: 0e7b9572c05e8fd8dbb6d80be1a9cd5d15ecabec
+  data.tar.gz: a7dc03f83063a2f39378ed99dedf96345440aaa7
 SHA512:
-  metadata.gz: a09768f1a182fa25eaae541c8af2e8396910eaccd1e9e08d313d7b5b98fe6aa098cd95de0c4dd88061683135b91a9c273f32489f8ab941cac2b74f4e0e1763cf
-  data.tar.gz: 611c0cb5a659adf051a0478bd9c621d2cc325d404db686e989217f6a9a755e2495e3697ff6b4f0999e6b38f9b8a5642f83842b6d456609647422ef48a53b15bc
+  metadata.gz: ac8b523cf687622f153b5778cb107dae447d5a68f9fb702cc71abb2240e4c62dc173665ac0a6cbd50a4955f03c2658ead475d1e7dd611d208d42c630c8ab4411
+  data.tar.gz: 560fea321249afbce734d49e633886fd33fa371b3e11cb4da9d9ae653f56262acc40c560d4044fb59bb14bc776a40ebcfa2db0b65cca0bd37342f80e67f4da0f

data/bin/rsrl

@@ -1,12 +1,23 @@
 #!/usr/bin/env ruby
+require 'pp'
+
 begin
   require File.join(".",File.dirname(__FILE__),'..','lib','sereal')
 rescue LoadError
   require 'sereal'
 end
 
+class SerealPerlObject
+  def inspect
+    "#{@class} - #{@value.inspect}"
+  end
+end
+
 content = ENV['Sereal_STREAM'] ? STDIN : ARGF.read
 compress = Sereal::RAW
+debug = ENV['Sereal_DEBUG'] ? Sereal::DEBUG : 0
+debug |= ENV['Sereal_REF'] ? Sereal::REF : 0
+debug |= ENV['Sereal_COPY'] ? Sereal::COPY : 0
 
 {'Sereal_SNAPPY' => Sereal::SNAPPY,'Sereal_SNAPPY_INCR' => Sereal::SNAPPY_INCR }.each do |k,v|
   if ENV[k]
@@ -15,10 +26,9 @@ compress = Sereal::RAW
 end
 
 if ENV['Sereal_STREAM'] || content[0..3] == '=srl'
-  Sereal.decode(content) do |x|
-    STDOUT.write(x)
-    puts
+  Sereal.decode(content,debug | Sereal::THAW) do |x|
+    PP.pp(x)
   end
 else
-  STDOUT.write(Sereal.encode(eval(content), compress))
+  STDOUT.write(Sereal.encode(eval(content), compress | debug))
 end

data/ext/sereal/buffer.h

@@ -3,7 +3,7 @@
 static void s_dump(sereal_t *s);
 
 static inline void s_free_data_if_not_mine(sereal_t *s) {
-    if (!(s->flags & FLAG_NOT_MINE)) {
+    if (!(s->flags & __NOT_MINE)) {
         if (s->data)
             free(s->data);
         s->data = NULL;
@@ -12,6 +12,7 @@ static inline void s_free_data_if_not_mine(sereal_t *s) {
 
 static inline void s_init_tracker(sereal_t *s) {
     if (s->tracked == Qnil) {
+        SD(s,"initializing tracker");
         s->tracked = rb_hash_new();
         rb_gc_mark(s->tracked);
     }
@@ -23,6 +24,13 @@ static inline void s_reset_tracker(sereal_t *s) {
     }
 }
 
+static inline void s_init_copy(sereal_t *s) {
+    if (s->copy == Qnil) {
+        s->copy = rb_hash_new();
+        rb_gc_mark(s->copy);
+    }
+}
+
 static inline void s_destroy(sereal_t *s) {
     if (!s)
         return;
@@ -46,6 +54,8 @@ static inline sereal_t * s_create(void) {
     sereal_t *s = s_alloc_or_raise(NULL,sizeof(*s));
     ZERO(s,sizeof(*s));
     s->tracked = Qnil;
+    s->copy = Qnil;
+
     return s;
 }
 
@@ -98,7 +108,7 @@ static inline int s_read_stream(sereal_t *s, u32 end) {
 static inline void *s_get_p_at_pos(sereal_t *s, u32 pos,u32 req) {
     // returning s->data[pos], so we just make size count from 0
     if (pos + req >= s->size) {
-        if (s->flags & FLAG_STREAM) {
+        if (s->flags & __STREAM) {
             if (s_read_stream(s,pos + req + 1) < 0) {
                 s_raise(s,rb_eRangeError,"stream request for %d bytes failed (err: %s)",
                         req,strerror(errno));
@@ -169,6 +179,11 @@ static inline u32 s_shift_position_bang(sereal_t *s, u32 len) {
     return len;
 }
 
+static inline void s_set_flag_at_pos(sereal_t *s, u32 pos, u8 flag) {
+    u8 *p = s_get_p_at_pos(s,pos,0);
+    *p |= flag;
+}
+
 static void b_dump(u8 *p, u32 len, u32 pos) {
     int i;
 

data/ext/sereal/decode.c

@@ -180,34 +180,112 @@ static VALUE s_read_extend(sereal_t *s, u8 tag) {
 }
 
 static VALUE s_read_ref(sereal_t *s, u8 tag) {
-    u64 off = s_get_varint_bang(s);
     if (s->tracked == Qnil)
         s_raise(s,rb_eArgError,"there are no references stored");
-    return rb_hash_aref(s->tracked,INT2FIX(off + s->hdr_end));
+    u64 off = s_get_varint_bang(s);
+    VALUE object = rb_hash_lookup(s->tracked,INT2FIX(off + s->hdr_end));
+    SD(s,"reading reference from offset: %d, id: %d",off + s->hdr_end,FIX2INT(rb_obj_id(object)));
+    return object;
 }
 
+#define TRAVEL(s,__stored)                                              \
+    do {                                                                \
+        u32 offset = s_get_varint_bang(s) - 1;                          \
+        __stored = s->pos;                                              \
+        s->pos = offset + s->hdr_end;                                   \
+        SD(s,"going back offset: %d, stored position: %d (tag: %d)",offset,stored_pos,tag); \
+    } while(0)
+#define BACK(s,__stored)                                \
+    do {                                                \
+        if (__stored > 0) {                             \
+            SD(s,"going forward to pos: %d",__stored);  \
+            s->pos = __stored;                          \
+        }                                               \
+    } while(0);
+
+
 static VALUE s_read_copy(sereal_t *s, u8 tag) {
-    VALUE ref = s_red_ref(s,tag);
-    return rb_obj_dup(ref);
+    u32 stored_pos = 0;
+    TRAVEL(s,stored_pos);
+    VALUE object = sereal_to_rb_object(s);
+    BACK(s,stored_pos);
+    return object;
+}
+
+
+#define MUST_BE_SOMETHING(__klass,__type)                               \
+    if (TYPE(__klass) != __type)                                        \
+        s_raise(s,rb_eTypeError,"unexpected object type: %s (expecting: %s(%d) got: %s(%d))",rb_obj_classname(__klass),(__type == T_STRING ? "String" : (__type == T_ARRAY ? "Array" : "_unknown_")),__type, rb_obj_classname(__klass),TYPE(__klass));
+
+static VALUE s_read_perl_object(sereal_t *s, u8 tag) {
+    u32 stored_pos = 0;
+    if (tag == SRL_HDR_OBJECTV)
+        TRAVEL(s,stored_pos);
+
+    VALUE s_klass = sereal_to_rb_object(s);
+    BACK(s,stored_pos);
+    MUST_BE_SOMETHING(s_klass,T_STRING);
+
+    SD(s,"fetched perl class named: %s",RSTRING_PTR(s_klass));
+
+    VALUE object = sereal_to_rb_object(s);
+
+    VALUE pobj =  rb_class_new_instance(0,NULL,SerealPerlObject);
+    rb_ivar_set(pobj,ID_CLASS,s_klass);
+    rb_ivar_set(pobj,ID_VALUE,object);
+    return pobj;
 }
 
+static VALUE s_read_object_freeze(sereal_t *s, u8 tag) {
+    if (!(s->flags & __THAW))
+        s_raise(s,rb_eTypeError,"object_freeze received, but decoder is initialized without Sereal::THAW option");
+
+    u32 stored_pos = 0;
+    if (tag == SRL_HDR_OBJECTV_FREEZE)
+        TRAVEL(s,stored_pos);
+
+    VALUE s_klass = sereal_to_rb_object(s);
+    BACK(s,stored_pos);
+    MUST_BE_SOMETHING(s_klass,T_STRING);
+
+    // hash it?
+    VALUE klass = rb_const_get(rb_cObject, rb_intern(RSTRING_PTR(s_klass)));
+    if (!rb_obj_respond_to(klass,THAW,0))
+        s_raise(s,rb_eTypeError,"class: %s does not respond to THAW",
+                rb_obj_classname(s_klass));
+
+    VALUE object = sereal_to_rb_object(s);
+    MUST_BE_SOMETHING(object,T_ARRAY);
+    rb_ary_unshift(object,ID2SYM(SEREAL));
+
+    return rb_funcall2(klass,THAW,RARRAY_LEN(object),RARRAY_PTR(object));
+}
+#undef TRAVEL
+#undef BACK
+
 VALUE sereal_to_rb_object(sereal_t *s) {
     u8 t, tracked;
     S_RECURSE_INC(s);
     u32 pos;
-    while (s->pos < s->size || (s->flags & FLAG_STREAM)) {
+    while (s->pos < s->size || (s->flags & __STREAM)) {
         t = s_get_u8_bang(s);
         tracked = (t & SRL_HDR_TRACK_FLAG ? 1 : 0);
         t &= ~SRL_HDR_TRACK_FLAG;
-        pos = s->pos;
 
-        S_RECURSE_DEC(s);
+        pos = s->pos;
 
         VALUE decoded = (*READERS[t])(s,t);
+
         if (tracked) {
             s_init_tracker(s);
-            rb_hash_aset(s->tracked,INT2FIX(pos),decoded);
+            SD(s,"tracking object of class: %s(id: %d) at position: %d",rb_obj_classname(decoded),FIX2INT(rb_obj_id(decoded)),pos);
+            VALUE v_pos = INT2FIX(pos);
+            if (rb_hash_lookup(s->tracked,v_pos) == Qnil)
+                rb_hash_aset(s->tracked,INT2FIX(pos),decoded);
         }
+
+        SD(s,"object: %s: %s",rb_obj_classname(decoded),RSTRING_PTR(rb_funcall(decoded,rb_intern("to_s"),0)));
+        S_RECURSE_DEC(s);
         return decoded;
     }
     s_raise(s,rb_eArgError,"bad packet, or broken decoder");
@@ -218,9 +296,19 @@ VALUE method_sereal_decode(VALUE self, VALUE args) {
     u32 argc = RARRAY_LEN(args);
     if (argc < 1)
         rb_raise(rb_eArgError,"need at least 1 argument (object)");
-    VALUE payload = rb_ary_shift(args);
+    VALUE payload = rb_ary_entry(args,0);
+
     u8 have_block = rb_block_given_p();
     sereal_t *s = s_create();
+    if (argc == 2) {
+        VALUE flags = rb_ary_entry(args,1);
+        if (flags != Qnil && flags != Qfalse) {
+            if (TYPE(flags) == T_FIXNUM)
+                s->flags = FIX2LONG(flags) & __ARGUMENT_FLAGS;
+            else
+                s_raise(s,rb_eArgError,"second argument must be an integer (used only for flags) %s given",rb_obj_classname(flags));
+        }
+    }
     u64 offset = 0;
 
     if (TYPE(payload) == T_FILE) {
@@ -229,8 +317,9 @@ VALUE method_sereal_decode(VALUE self, VALUE args) {
 
         rb_io_t *fptr;
         GetOpenFile(payload, fptr);
-        s->flags |= FLAG_STREAM;
+        s->flags |= __STREAM;
         s->fd = fptr->fd;
+        SD(s,"reading strea with fd: %d",s->fd);
     } else if (TYPE(payload) != T_STRING) {
         rb_raise(rb_eTypeError,"can not decode objects of type %s",rb_obj_classname(payload));
     }
@@ -239,14 +328,13 @@ again:
     s->pos = 0;
     s_reset_tracker(s);
 
-    if (s->flags & FLAG_STREAM) {
+    if (s->flags & __STREAM) {
         s->size = 0;
         s->rsize = 0;
         if (s_read_stream(s,__MIN_SIZE) < 0) {
             s_destroy(s);
             return Qnil;
         }
-
     } else {
         u32 size = RSTRING_LEN(payload) - offset;
         if (offset > RSTRING_LEN(payload) || (offset > 0 && size < __MIN_SIZE)) {
@@ -256,7 +344,7 @@ again:
         if (size < __MIN_SIZE)
             s_raise(s,rb_eTypeError,"size(%d) is less then min packet size %d, offset: %d",size,__MIN_SIZE,offset);
 
-        s->flags |= FLAG_NOT_MINE;
+        s->flags |= __NOT_MINE;
         s->data = RSTRING_PTR(payload) + offset;
         s->size = size;
     }
@@ -269,12 +357,15 @@ again:
     u8 suffix = s_get_varint_bang(s);
     u8 is_compressed;
 
-    if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY)
+    if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY) {
         is_compressed = __SNAPPY;
-    else if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY_INCR)
+    } else if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY_INCR) {
         is_compressed = __SNAPPY_INCR;
-    else
+    } else {
         is_compressed = __RAW;
+    }
+
+    SD(s,"initialized (s) with compression type: %d",is_compressed);
 
     if (is_compressed) {
         u32 uncompressed_len;
@@ -283,13 +374,13 @@ again:
         if (is_compressed == __SNAPPY_INCR) {
             compressed_len = s_get_varint_bang(s);
         } else {
-            if (s->flags & FLAG_STREAM)
+            if (s->flags & __STREAM)
                 s_raise(s,rb_eTypeError,"parsing non incremental compressed objects, from stream of data, is not supported");
 
             compressed_len = s->size - s->pos;
         }
-
-        if (s->flags & FLAG_STREAM)
+        SD(s,"compressed len: %d",compressed_len);
+        if (s->flags & __STREAM)
             s_get_p_req_inclusive(s,compressed_len);
 
         int snappy_header_len = csnappy_get_uncompressed_length(s_get_p_req_inclusive(s,compressed_len),
@@ -312,9 +403,11 @@ again:
         s->size = uncompressed_len;
         offset += s->pos + compressed_len;
         s->pos = 0;
-        s->flags &= ~FLAG_NOT_MINE;
+        s->flags &= ~__NOT_MINE;
     }
+
     s->hdr_end = s->pos;
+    SD(s,"header end at %d",s->hdr_end);
     VALUE result = sereal_to_rb_object(s);
     if (!is_compressed)
         offset += s->pos;

data/ext/sereal/decode.h

@@ -23,6 +23,8 @@ static VALUE s_read_pad(sereal_t *s, u8 tag);
 static VALUE s_read_extend(sereal_t *s, u8 tag);
 static VALUE s_read_ref(sereal_t *s, u8 tag);
 static VALUE s_read_copy(sereal_t *s, u8 tag);
+static VALUE s_read_object_freeze(sereal_t *s, u8 tag);
+static VALUE s_read_perl_object(sereal_t *s, u8 tag);
 
 static VALUE (*READERS[256])(sereal_t *, u8) = {
         s_read_small_positive_int,                      // 0    SRL_HDR_POS_LOW
@@ -69,14 +71,14 @@ static VALUE (*READERS[256])(sereal_t *, u8) = {
         s_read_ref,                                     // 41   SRL_HDR_REFP
         s_read_hash,                                    // 42   SRL_HDR_HASH
         s_read_array,                                   // 43   SRL_HDR_ARRAY
-        s_default_reader, /* XXX */                     // 44   SRL_HDR_OBJECT
-        s_default_reader, /* XXX */                     // 45   SRL_HDR_OBJECTV
+        s_read_perl_object,                             // 44   SRL_HDR_OBJECT
+        s_read_perl_object,                             // 45   SRL_HDR_OBJECTV
         s_read_ref,                                     // 46   SRL_HDR_ALIAS
         s_read_copy,                                    // 47   SRL_HDR_COPY
         s_default_reader, /* XXX */                     // 48   SRL_HDR_WEAKEN
         s_read_regexp,                                  // 49   SRL_HDR_REGEXP
-        s_default_reader, /* XXX */                     // 50   SRL_HDR_RESERVED_LOW
-        s_default_reader, /* XXX */                     // 51
+        s_read_object_freeze,                           // 50   OBJECT_FREEZE
+        s_read_object_freeze,                           // 51   OBJECTV_FREEZE
         s_default_reader, /* XXX */                     // 52
         s_default_reader, /* XXX */                     // 53
         s_default_reader, /* XXX */                     // 54

data/ext/sereal/encode.c

@@ -28,6 +28,12 @@
 #elif T_NIL > W_SIZE
 	#define W_SIZE T_NIL
 #endif
+#define COMPLEX(object)                         \
+    (TYPE(object) == T_ARRAY  ||                \
+     TYPE(object) == T_HASH   ||                \
+     TYPE(object) == T_SYMBOL ||                \
+     TYPE(object) == T_OBJECT ||                \
+     TYPE(object) == T_STRING)
 
 /* function pointer array */
 void (*WRITER[W_SIZE])(sereal_t *,VALUE);
@@ -146,15 +152,57 @@ static void s_append_hash(sereal_t *s, VALUE object) {
 	convert symbols to strings
 */
 static void s_append_symbol(sereal_t *s, VALUE object) {
-	VALUE string = rb_sym_to_s(object);
-	s_append_rb_string(s,string);
+    VALUE string = rb_sym_to_s(object);
+    s_append_rb_string(s,string);
 }
 
+static void s_append_copy(sereal_t *s, VALUE object) {
+    u32 pos = FIX2LONG(object);
+    s_append_hdr_with_varint(s,SRL_HDR_COPY,pos - s->hdr_end + 1);
+}
+
+static VALUE s_copy_or_keep_in_mind(sereal_t *s, VALUE object) {
+    if (s->copy == Qnil)
+        return Qnil;
+
+    VALUE stored_position = rb_hash_lookup(s->copy,object);
+    if (stored_position == Qnil)
+        rb_hash_aset(s->copy,object,INT2FIX(s->pos));    
+    return stored_position;
+}
+
+
 /*
-	call object.to_srl and serialize the result
+        try to FREEZE the object so it can be THAW-ed at decode time
+        if not possible (no Sereal::THAW argument or object does not
+        repsond to FREEZE), just call object.to_srl and serialize the
+        result
 */
 static void s_append_object(sereal_t *s, VALUE object) {
-	rb_object_to_sereal(s,rb_funcall(object,rb_intern("to_srl"),0));
+    if (s->flags & __THAW && rb_obj_respond_to(object,FREEZE,0)) {
+        VALUE klass = rb_class_name(CLASS_OF(object));
+        VALUE copy = s_copy_or_keep_in_mind(s,klass);
+        if (copy != Qnil) {
+            s_append_u8(s,SRL_HDR_OBJECTV_FREEZE);
+            s_append_copy(s,copy);
+        } else {
+            s_append_u8(s,SRL_HDR_OBJECT_FREEZE);
+            s_append_rb_string(s,rb_class_name(CLASS_OF(object)));
+        }
+        VALUE frozen = rb_funcall(object,FREEZE,1,ID2SYM(SEREAL));
+        if (TYPE(frozen) != T_ARRAY)
+            s_raise(s,rb_eTypeError,"Sereal spec requires FREEZE to return array instead %s",rb_obj_classname(frozen));
+
+        // REFN + ARRAY
+        s_append_u8(s,SRL_HDR_REFN);
+        s_append_hdr_with_varint(s,SRL_HDR_ARRAY,RARRAY_LEN(frozen));
+        int i;
+        for (i = 0; i < RARRAY_LEN(frozen); i++)
+            rb_object_to_sereal(s,rb_ary_entry(frozen,i));
+
+    } else {
+        rb_object_to_sereal(s,rb_funcall(object,TO_SRL,0));
+    }
 }
 
 
@@ -219,31 +267,31 @@ static void s_append_nil(sereal_t *s, VALUE object) {
 static void s_append_refp(sereal_t *s, VALUE object) {
     u32 pos = FIX2LONG(object);
     s_append_hdr_with_varint(s,SRL_HDR_REFP,pos - s->hdr_end + 1);
-    u8 *reference = s_get_p_at_pos(s,pos,0);
-    *reference |= SRL_HDR_TRACK_FLAG;
+    s_set_flag_at_pos(s,pos,SRL_HDR_TRACK_FLAG);
 }
 
 /* writer function pointers */
 static void rb_object_to_sereal(sereal_t *s, VALUE object) {
     S_RECURSE_INC(s);
     u32 pos = s->pos;
-
-    if (s->tracked != Qnil &&
-        TYPE(object) == T_ARRAY  ||
-        TYPE(object) == T_HASH   ||
-        TYPE(object) == T_SYMBOL ||
-        TYPE(object) == T_STRING) {
-
+    if (COMPLEX(object)) {
+        VALUE stored;
         if (s->tracked != Qnil) {
-            VALUE id = rb_obj_id(object);
-            VALUE stored_position = rb_hash_aref(s->tracked,id);
-            if (stored_position != Qnil) {
-                s_append_refp(s,stored_position);
-                goto out;
-            } else {
+            if (s->tracked != Qnil) {
+                VALUE id = rb_obj_id(object);
+                stored = rb_hash_lookup(s->tracked,id);
+                if (stored != Qnil) {
+                    s_append_refp(s,stored);
+                    goto out;
+                }
                 rb_hash_aset(s->tracked,id,INT2FIX(pos));
             }
         }
+        stored = s_copy_or_keep_in_mind(s,object);
+        if (stored != Qnil) {
+            s_append_copy(s,stored);
+            goto out;
+        }
     }
 
     (*WRITER[TYPE(object)])(s,object);
@@ -267,16 +315,21 @@ void fixup_varint_from_to(u8 *varint_start, u8 *varint_end, u32 number) {
     }
 }
 
-VALUE method_sereal_encode(VALUE self, VALUE args) {
+
+/*
+ * Encodes object into Sereal
+ */
+VALUE
+method_sereal_encode(VALUE self, VALUE args) {
     u32 argc = RARRAY_LEN(args);
     if (argc < 1)
         rb_raise(rb_eArgError,"need at least 1 argument (object)");
 
     sereal_t *s = s_create();
-    VALUE payload = rb_ary_shift(args);
+    VALUE payload = rb_ary_entry(args,0);
     VALUE compress = Qfalse;
     if (argc == 2)
-        compress = rb_ary_shift(args);
+        compress = rb_ary_entry(args,1);
 
     u8 do_compress;
     u8 version = SRL_PROTOCOL_VERSION;
@@ -286,10 +339,15 @@ VALUE method_sereal_encode(VALUE self, VALUE args) {
     } else {
         do_compress = (compress == Qtrue ? 1 : 0);
     }
-    if (do_compress & __REF) {
-        do_compress &= ~__REF;
+
+    s->flags = do_compress & __ARGUMENT_FLAGS;
+    do_compress &=~ __ARGUMENT_FLAGS;
+    if (s->flags & __REF)
         s_init_tracker(s);
-    }
+
+    if (s->flags & __COPY)
+        s_init_copy(s);
+
     switch(do_compress) {
         case __SNAPPY:
             version |= SRL_PROTOCOL_ENCODING_SNAPPY;

data/ext/sereal/proto.h

@@ -33,16 +33,19 @@
 #define SRL_HDR_ARRAY           ((char)43)      /* <COUNT-VARINT> [<ITEM-TAG> ...] - count followed by items */
 #define SRL_HDR_OBJECT          ((char)44)      /* <STR-TAG> <ITEM-TAG> - class, object-item */
 #define SRL_HDR_OBJECTV         ((char)45)      /* <OFFSET-VARINT> <ITEM-TAG> - offset of previously used classname tag - object-item */
+
 #define SRL_HDR_ALIAS           ((char)46)      /* <OFFSET-VARINT> - alias to item defined at offset */
 #define SRL_HDR_COPY            ((char)47)      /* <OFFSET-VARINT> - copy of item defined at offset */
 
 #define SRL_HDR_WEAKEN          ((char)48)      /* <REF-TAG> - Weaken the following reference */
 #define SRL_HDR_REGEXP          ((char)49)      /* <PATTERN-STR-TAG> <MODIFIERS-STR-TAG>*/
+#define SRL_HDR_OBJECT_FREEZE   ((char)50)      /* <STR-TAG> <ITEM-TAG> - class, object-item. Need to call "THAW" method on class after decoding */
+#define SRL_HDR_OBJECTV_FREEZE  ((char)51)      /* <OFFSET-VARINT> <ITEM-TAG> - (OBJECTV_FREEZE is to OBJECT_FREEZE as OBJECTV is to OBJECT) */
 
 /* Note: Can do reserved check with a range now, but as we start using
  *       them, might have to explicit == check later. */
-#define SRL_HDR_RESERVED        ((char)50)      /* reserved */
-#define SRL_HDR_RESERVED_LOW    ((char)50)
+#define SRL_HDR_RESERVED        ((char)52)      /* reserved */
+#define SRL_HDR_RESERVED_LOW    ((char)52)
 #define SRL_HDR_RESERVED_HIGH   ((char)57)
 
 #define SRL_HDR_FALSE           ((char)58)      /* false (PL_sv_no)  */

data/ext/sereal/sereal.c

@@ -2,36 +2,44 @@
 #include "encode.h"
 
 VALUE Sereal = Qnil;
+VALUE SerealPerlObject = Qnil;
+ID FREEZE;
+ID THAW;
+ID TO_SRL;
+ID SEREAL;
+ID ID_CLASS;
+ID ID_VALUE;
 void Init_sereal();
 
 /*
  * Encode/Decode object using Sereal binary protocol:
  * https://github.com/Sereal/Sereal/blob/master/sereal_spec.pod
  *
- *   Sereal.encode(object) -> serialized blob
- *   Sereal.encode(object,Sereal::SNAPPY_INCR) -> snappy compressed blob
- *   Sereal.encode(object,Sereal::SNAPPY) -> snappy compressed blob
+ * ==install:
  *
- * SNAPPY_INCR encoded objects can be appended into one output and then the
- * decoder will know what to do.
+ *  $ gem install sereal
  *
- * Sereal.encode(object,Sereal::REF)
- * or Sereal::REF|Sereal::SNAPPY_INC, or Sereal::REF|Sereal::SNAPPY
+ * or you can build it from github which requires: 
+ * 1. rake compiler - <code>gem install rake-compiler</code> (https://github.com/luislavena/rake-compiler)
+ * 2. ruby 1.9+ or rubinius supporting 1.9+
  *
- * when encoding will try to use Sereal's REFP tag to transmit only the
- * the original object's offset in the packet.
- * So:
- *    one = [ 1,2,3,4,5 ]
- *    two = [ one, one ]
- *    Sereal.encode(two,Sereal::REF)
- * will send 'one' only once, and one REFP that points to the first one
- * it uses one.object_id as a hash key in a local tracker hash
- * and if it sees this object_id again it just sends the offset.
+ *  $ git clone https://github.com/Sereal/Sereal
+ *  $ cd Sereal/ruby  
+ *  $ gem build sereal.gemspec
+ *  $ gem install sereal-0.0.?.gem
  *
+ * ==serialize:
+ *   require 'sereal'
+ *   Sereal.encode(object)
  *
- *   Sereal.decode(blob) - returns the decoded object
- *   
- * If the blob contains multiple compressed
+ * ===serializing objects
+ * if Sereal::THAW option is given the encoder will try to call FREEZE() instance method on
+ * the object beeing serialized, and it will serialize the class name + the output of FREEZE (look the THAW constant for more information)
+ * if the object does not respond to FREEZE it will call <code>to_srl</code> and serialize the result of that
+ * ==deserialize:
+ *   require 'sereal'
+ *   Sereal.decode(blob) 
+ * If the blob contains multiple compressed objects
  * sub-blobs you should call it with:
  *
  *    Sereal.decode(blob) do |decoded|
@@ -39,6 +47,7 @@ void Init_sereal();
  *    end
  *
  * otherwise only the first decoded object will be returned
+ * ===stream decoding
  * there is also streaming support which takes any kind of IO object
  * like socket, or just regular File, and it is really easy to use:
  *
@@ -46,7 +55,7 @@ void Init_sereal();
  *      # do something with the decoded object
  *    end
  *
- * it works both with `incremental snappy` and with just combined sereal packets.
+ * it works both with SNAPPY_INCR and with just combined sereal packets.
  * another example but with TCPSocket:
  *
  *    s = TCPSocket.new 'localhost', 2000
@@ -54,15 +63,216 @@ void Init_sereal();
  *      # do something with the decoded object
  *    end
  *
+ * ===multiple packets in one buffer
+ * it also supports decoding of multiple packets in one buffer:
+ *
+ *    buf = ""
+ *    buf << Sereal.encode([1,2,3],Sereal::SNAPPY_INCR)
+ *    buf << Sereal.encode([3,4,5])
+ *    buf << Sereal.encode([7,8,9],Sereal::SNAPPY_INCR)
+ *    Sereal.decode(buf) do |decoded|
+ *      p decoded
+ *    end
+ * ==Sereal.encode() and Sereal.decode() accept:
+ * 1. compression types: RAW, SNAPPY_INCR, and SNAPPY
+ * 2. flags: REF, COPY, THAW and DEBUG
+ *
+ * flags and compression types can be used in combinations like:
+ *
+ *    Sereal.encode([1,2,3],Sereal::REF|Sereal::COPY|Sereal::THAW|Sereal::SNAPPY_INCR)
+ *
+ * but you can not use 2 types of compression in the same time
+ *
+ * ==LZ4
+ * For brief period (version 0.0.5 to 0.0.6) there was a support for LZ4 and LZ4HC, which was pushed to the master branch by mistake. if you are depending on it please convert yout data using <code>bin/rsrl</code> or just use <code>0.0.5</code> version of the sereal gem.
+ *
+ *    gem 'sereal', '= 0.0.5'
+ *    #or
+ *    $ gem install sereal -v 0.0.5
+ *
  */
 void Init_sereal() {
-        Sereal = rb_define_class("Sereal", rb_cObject);
-        rb_define_singleton_method(Sereal, "encode", method_sereal_encode, -2);
-        rb_define_singleton_method(Sereal, "decode", method_sereal_decode, -2);
-        rb_define_const(Sereal, "SNAPPY",INT2NUM(__SNAPPY));
-        rb_define_const(Sereal, "SNAPPY_INCR",INT2NUM(__SNAPPY_INCR));
-        rb_define_const(Sereal, "RAW",INT2NUM(__RAW));
-        rb_define_const(Sereal, "REF",INT2NUM(__REF));
-        s_init_writers();
+    TO_SRL = rb_intern("to_srl");
+    THAW = rb_intern("THAW");
+    FREEZE = rb_intern("FREEZE");
+    SEREAL = rb_intern("Sereal");
+    ID_CLASS = rb_intern("@class");
+    ID_VALUE = rb_intern("@value");
+
+    SerealPerlObject = rb_define_class("SerealPerlObject", rb_cObject);
+
+    Sereal = rb_define_class("Sereal", rb_cObject);
+    rb_define_singleton_method(Sereal, "encode", method_sereal_encode, -2);
+
+    rb_define_singleton_method(Sereal, "decode", method_sereal_decode, -2);
+
+    /*
+     * instructs the encoder to use Snappy compression
+     * nb: this is Sereal protocol version 1 only
+     * do not use it if possible.
+     * 
+     *    Sereal.encode(object,Sereal::SNAPPY)
+     */
+    rb_define_const(Sereal, "SNAPPY",INT2NUM(__SNAPPY));
+
+
+    /* 
+     * instructs the encoder to use Snappy compression
+     * but with support for incremental packet (meaning
+     * you can combine packets into one big blob of data
+     * and the encoder will be confident that there is no
+     * corruption, because the SNAPPY_INCR packet contains the
+     * uncompressed length)
+     *
+     *    Sereal.encode(object,Sereal::SNAPPY_INCR)
+     */
+    rb_define_const(Sereal, "SNAPPY_INCR",INT2NUM(__SNAPPY_INCR));
+
+    /* 
+     * intructs the encoder to use no compression (default)
+     *
+     *    Sereal.encode(object,Sereal::RAW)
+     */
+    rb_define_const(Sereal, "RAW",INT2NUM(__RAW));
+
+    /*
+     * (can also be used with any compression type/RAW)
+     * instructs the encoder to keep track of object_ids
+     * and when it sees that object with the same id has
+     * already been encoded, it just creates REFP reference
+     * with offset to the first item
+     * so:
+     *    name = "john doe"
+     *    object = [ name, name ]
+     *    Sereal.encode(name,Sereal::REF|Sereal::SNAPPY_INCR)
+     *
+     * will actually create something that looks like:
+     *
+     *    000006/000001: 42  066 ARRAYREF(2)
+     *    000007/000002: 68* 232   SHORT_BINARY(8): 'john doe'
+     *    000016/000011: 29  041   REFP(2)
+     * 
+     * instead of:
+     * 
+     *    000006/000001: 42  066 ARRAYREF(2)
+     *    000007/000002: 68  104   SHORT_BINARY(8): 'john doe'
+     *    000016/000011: 68  104   SHORT_BINARY(8): 'john doe'
+     *
+     * as you can see Sereal saved us 7 bytes
+     *
+     * this can hurts performance because the encoder must get the
+     * object_id of every encoded object, and put it in a hash
+     * with the current position, so it can be looked up later
+     */
+    rb_define_const(Sereal, "REF",INT2NUM(__REF));
+
+
+    /* 
+     * very similar to Sereal::REF, but it instructs the decoder
+     * to create new item, by going back in time as if the OFFSET
+     * of the COPY tag was beeing read right now
+     * it puts every object as a key in a hash (instead of its 
+     * object_id)
+     * 
+     *    object = [ { name => "john"} , { name => "john"} ]
+     *    Sereal.encode(name,Sereal::COPY|Sereal::SNAPPY_INCR)
+     *
+     * will produce:
+     *
+     *    000006/000001: 42  066 ARRAYREF(2)
+     *    000007/000002: 51  081   HASHREF(2)
+     *        KEY:
+     *    000008/000003: 64  100       SHORT_BINARY(4): 'name'
+     *        VALUE:
+     *    000013/000008: 64  100       SHORT_BINARY(4): 'john'
+     *    000018/000013: 2f  047   COPY(2)
+     *
+     * as you can see the hash is sent only once
+     * 
+     * COPY can be used with REF like:
+     * 
+     *    object = "bazinga"
+     *    
+     *    Sereal.encode([object,"bazinga",object],Sereal::REF|Sereal::COPY)
+     *
+     * will produce:
+     *    000006/000001: 43  067 ARRAYREF(3)
+     *    000007/000002: 67* 231   SHORT_BINARY(7): 'bazinga'
+     *    000015/000010: 2f  047   COPY(2)
+     *    000017/000012: 29  041   REFP(2)
+     *
+     * using Sereal::COPY also hurts performance because every encoding step
+     * the encoder must look into a hash if the object exists in it
+     * so it can create a COPY tag instead of encoding the object again
+     */
+    rb_define_const(Sereal, "COPY",INT2NUM(__COPY));
+
+    /*
+     * add support for FREEZE/THAW
+     * it calls <code>object.FREEZE(:Sereal)</code> and it serializes the result of that
+     * with the object's class name, when deserializing it calls 
+     * <code>class.THAW(:Sereal,the output of FREEZE)</code>
+     * 
+     *    class StorableFile
+     *      attr_accessor :path
+     *      def initialize(path,pos)
+     *        @path = path
+     *        @pos = pos
+     *      end
+     *      def read
+     *        @pos += 1
+     *      end
+     *      def FREEZE(serializer)
+     *        [@path,@pos]
+     *      end
+     *      def self.THAW(serializer,path,pos)
+     *        self.new(path,pos)
+     *      end
+     *    end
+     * 
+     *    obj = StorableFile.new("/tmp/sereal.txt",0)
+     *    obj.read # read some data
+     *    encoded = Sereal.encode(obj,Sereal::THAW)
+     *    # this actually encodes something like:
+     *    #   'StorableFile' -> [ "/tmp/sereal.txt", 1 ]
+     *    restored = Sereal.decode(encoded,Sereal::THAW)
+     *    # this will call Sereal.THAW(:Sereal,"/tmp/sereal.txt",1)
+     *
+     * as you can see the array returned from FREEZE is exploded into
+     * arguments for THAW, FREEZE *MUST* return array, otherwise the
+     * encoder throws TypeException.
+     * *BOTH* encoder *AND* decoder must be started with Sereal::THAW option
+     * in order for it to work properly
+     */
+    rb_define_const(Sereal, "THAW",INT2NUM(__THAW));
+    
+    /*
+     * the argument given to FREEZE() and THAW() as serializer
+     * which at the moment is:
+     *
+     *    :Sereal
+     */
+    rb_define_const(Sereal, "FREEZER",ID2SYM(SEREAL));
+
+    /*
+     * enable debug output
+     *
+     *    name = "john"
+     *    Sereal.decode([name,name],Sereal::REF|Sereal::DEBUG)
+     *
+     * procudes:
+     *
+     *    initialized (s) with compression type: 0 { p: 6, s: 14, l: 0, h: 0 }  method_sereal_decode()
+     *    header end at 6 { p: 6, s: 14, l: 0, h: 6 }  method_sereal_decode()
+     *      tracking object of class: String(id: 7678920) at position: 8 { p: 12, s: 14, l: 2, h: 6 }  sereal_to_rb_object()
+     *      object: String: john { p: 12, s: 14, l: 2, h: 6 }  sereal_to_rb_object()
+     *      reading reference from offset: 8, id: 7678920 { p: 14, s: 14, l: 2, h: 6 }  s_read_ref()
+     *      object: String: john { p: 14, s: 14, l: 2, h: 6 }  sereal_to_rb_object()
+     *     object: Array: ["john", "john"] { p: 14, s: 14, l: 1, h: 6 }  sereal_to_rb_object()
+     *
+     */ 
+    rb_define_const(Sereal, "DEBUG",INT2NUM(__DEBUG));
+
+    s_init_writers();
 }
 

data/ext/sereal/sereal.h

@@ -12,7 +12,6 @@ typedef unsigned int            u32;
 typedef unsigned short          u16;
 typedef unsigned char           u8;
 typedef struct _sereal          sereal_t;
-typedef struct _track_entry     track_t;
 
 #define TRUE 1
 #define FALSE 0
@@ -41,19 +40,27 @@ typedef struct _track_entry     track_t;
 #define EXTENDED RE_OPTION_EXTENDED
 #endif
 
-#define FORMAT(fmt,arg...) fmt " [%s():%s:%d @ %u]\n",##arg,__func__,__FILE__,__LINE__,(unsigned int) time(NULL)
+#define FORMAT(fmt,arg...) fmt " %s()\n",##arg,__func__
 #define E(fmt,arg...) fprintf(stderr,FORMAT(fmt,##arg))
 #define D(fmt,arg...) printf(FORMAT(fmt,##arg))
+#define SD(s,fmt,arg...)                                                \
+    do {                                                                \
+        if (s->flags & __DEBUG) {                                       \
+            int i;                                                      \
+            for (i = 0; i < s->level; i++) {                            \
+                printf(" ");                                            \
+            }                                                           \
+            D(fmt " { p: %d, s: %d, l: %u, h: %u } ",##arg,s->pos,s->size,s->level,s->hdr_end); \
+        }                                                               \
+    } while(0);
 
 #define s_raise(what,ex,arg...)         \
 do {                                    \
+    SD(s,"s_raise");                    \
     s_destroy(what);                    \
     rb_raise(ex,##arg);                 \
 } while(0);
 
-#define FLAG_NOT_MINE 1
-#define FLAG_STREAM   2
-#define FLAG_REF      4
 struct _sereal {
     u8 *data;
     u32 size;
@@ -61,7 +68,9 @@ struct _sereal {
     u32 rsize;
     u32 level;
     u8 flags;
+    u8 expect;
     VALUE tracked;
+    VALUE copy;
     u32 hdr_end;
     int fd;
     struct buffer {
@@ -73,13 +82,20 @@ struct _sereal {
 
 VALUE method_sereal_encode(VALUE self, VALUE args);
 VALUE method_sereal_decode(VALUE self, VALUE payload);
+extern ID FREEZE;
+extern ID THAW;
+extern ID TO_SRL;
+extern ID SEREAL;
+extern ID ID_CLASS;
+extern ID ID_VALUE;
+extern VALUE SerealPerlObject;
 
-#define S_RECURSE_INC(s)                                        \
-    do {                                                        \
-        if((s)->level++ > MAX_RECURSION_DEPTH)                  \
-        s_raise((s),rb_eArgError,                               \
-                "max recursion depth reached: %d (level: %d)",  \
-                MAX_RECURSION_DEPTH, s->level);                 \
+#define S_RECURSE_INC(s)                                            \
+    do {                                                            \
+        if((s)->level++ > MAX_RECURSION_DEPTH)                      \
+            s_raise((s),rb_eArgError,                               \
+                    "max recursion depth reached: %d (level: %d)",  \
+                    MAX_RECURSION_DEPTH, s->level);                 \
     } while(0);
 
 #define S_RECURSE_DEC(s) ((s)->level--)
@@ -94,5 +110,12 @@ VALUE method_sereal_decode(VALUE self, VALUE payload);
 #define __SNAPPY        1
 #define __SNAPPY_INCR   2
 #define __REF           4
+#define __DEBUG         8
+#define __NOT_MINE      16
+#define __STREAM        32
+#define __THAW          64
+#define __COPY          64
+#define __ARGUMENT_FLAGS (__DEBUG|__THAW|__REF|__COPY)
+
 #define __MIN_SIZE      6
 #endif

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sereal
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Borislav Nikolov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-03 00:00:00.000000000 Z
+date: 2014-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -30,7 +30,8 @@ executables:
 - rsrl
 extensions:
 - ext/sereal/extconf.rb
-extra_rdoc_files: []
+extra_rdoc_files:
+- ext/sereal/sereal.c
 files:
 - ext/sereal/decode.c
 - ext/sereal/encode.c
@@ -52,7 +53,9 @@ homepage: https://github.com/Sereal/Sereal
 licenses: []
 metadata: {}
 post_install_message: 
-rdoc_options: []
+rdoc_options:
+- --exclude
+- .*\.so
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement