sereal 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 658547490d95c374a8fad16740ca8bdb463394cf
-  data.tar.gz: ff7871ff2e838a756d0a62c7094302068355e89f
+  metadata.gz: 06153f3bc1df8650c81eff2d5ef44f260920830c
+  data.tar.gz: e6e3e806787c10d2511ff7ea63256f922a45b0f3
 SHA512:
-  metadata.gz: af647c218413b546b956392a909e97dc54302a5c729ed2b0f45eeec8c2431b611c7403c343025c9913421a5ebcda477921d5452f080bd02b29c47f93438f4f2e
-  data.tar.gz: fff160c78207616f73f61380d78384b148053a89bf2b31c2519bfda961fb2dbb261772517f87f0b977cf7c2a17172b8a99fa27c94c42519fc141855b90294f7f
+  metadata.gz: 278c31a6754bd905858874e88d3cd87ed88986dbec21e3d7a5544d8b3fd7fab2d011ac3f7e22aff808a412e5b900e8f8d12427e8c1b3d05823cba271075b7379
+  data.tar.gz: a10efcc28da1409da425e6e6c222712fdaa7d62bee525728d9ea9b1edfc2a34ea0b14cb358fd98540063b4af1623f5e3b5995d0f17c251f044e67243513bea21

data/bin/rsrl

@@ -1,11 +1,11 @@
 #!/usr/bin/env ruby
 begin
-  require './lib/sereal'
+  require File.join(File.dirname(__FILE__),'..','lib','sereal')
 rescue LoadError
   require 'sereal'
 end
 
-content = ARGF.read
+content = ENV['Sereal_STREAM'] ? STDIN : ARGF.read
 compress = Sereal::RAW
 
 {'Sereal_SNAPPY' => Sereal::SNAPPY,'Sereal_SNAPPY_INCR' => Sereal::SNAPPY_INCR }.each do |k,v|
@@ -13,9 +13,11 @@ compress = Sereal::RAW
     compress = v
   end
 end
-if content[0..3] == '=srl'
+
+if ENV['Sereal_STREAM'] || content[0..3] == '=srl'
   Sereal.decode(content) do |x|
     STDOUT.write(x)
+    puts
   end
 else
   STDOUT.write(Sereal.encode(eval(content), compress))

data/ext/sereal/buffer.h

@@ -1,105 +1,181 @@
 #include "sereal.h"
-static inline void *realloc_or_raise(void *p, u32 n) {
-        u8 *buf = realloc(p,n);
-        if (!buf)
-                rb_raise(rb_eNoMemError,"memory allocation failure for %d bytes",n);
-        return buf;
-}
-static inline void *alloc_or_raise(u32 s) {
-        return realloc_or_raise(NULL,s);
+#include <errno.h>
+static void s_dump(sereal_t *s);
+
+static inline void s_free_data_if_not_mine(sereal_t *s) {
+    if (!(s->flags & FLAG_NOT_MINE)) {
+        if (s->data)
+            free(s->data);
+        s->data = NULL;
+    }
 }
 
-static inline sereal_t * s_create(void) {
-        sereal_t *s = alloc_or_raise(sizeof(*s));
-	ZERO(s,sizeof(*s));
-        return s;
+static inline void s_reset_tracker(sereal_t *s) {
+    if (s->tracked != Qnil)
+        rb_gc_unregister_address(&s->tracked);
+    s->tracked = Qnil;
 }
 
 static inline void s_destroy(sereal_t *s) {
-        if (s->tracked != Qnil)
-                rb_gc_unregister_address(&s->tracked);
-        if (s->data && (s->flags & FLAG_NOT_MINE) == 0)
-                free(s->data);
+    if (!s)
+        return;
+
+    s_reset_tracker(s);
+    s_free_data_if_not_mine(s);
+    free(s);
+}
+
+static inline void *s_realloc_or_raise(sereal_t *s, void *p, u32 n) {
+    u8 *buf = realloc(p,n);
+    if (!buf)
+        s_raise(s,rb_eNoMemError,"memory allocation failure for %d bytes",n);
+    return buf;
+}
+static inline void *s_alloc_or_raise(sereal_t *s,u32 n) {
+    return s_realloc_or_raise(s,NULL,n);
+}
 
-        free(s);
+static inline sereal_t * s_create(void) {
+    sereal_t *s = s_alloc_or_raise(NULL,sizeof(*s));
+    ZERO(s,sizeof(*s));
+    s->tracked = Qnil;
+    return s;
 }
 
 static inline void s_alloc(sereal_t *s, u32 len) {
-        if (s->rsize > s->size + len)
-                return;
+    if (s->rsize > s->size + len)
+        return;
 
-        u32 size = s->size + len + 512; // every time allocate 512b more, so we wont alloc for a while
-        u8 *buf = realloc_or_raise(s->data,size); 
-        s->data = buf;
-        s->rsize = size;
+    u32 size = s->size + len + 512; // every time allocate 512b more, so we wont alloc for a while
+    u8 *buf = s_realloc_or_raise(s,s->data,size); 
+    s->data = buf;
+    s->rsize = size;
 }
 
 static inline void s_append(sereal_t *s, void *suffix, u32 s_len) {
-        s_alloc(s,s_len);
-        COPY(suffix,(s->data + s->size), s_len);
-        s->size += s_len;
-        s->pos += s_len;
+    s_alloc(s,s_len);
+    COPY(suffix,(s->data + s->size), s_len);
+    s->size += s_len;
+    s->pos += s_len;
 }
 
 static inline void s_append_u8(sereal_t *s,u8 b) {
-        s_append(s,&b,sizeof(b));
+    s_append(s,&b,sizeof(b));
 }
 
 static inline void s_append_u32(sereal_t *s,u32 b) {
-        s_append(s,&b,sizeof(b));
+    s_append(s,&b,sizeof(b));
+}
+
+static inline int s_read_stream(sereal_t *s, u32 end) {
+     u32 pos = s->pos;
+     while (s->size < end) {
+        u32 req = end - s->size;
+        u32 left = s->buffer.size - s->buffer.pos;
+        if (left == 0) {
+            int rc = read(s->fd,s->buffer.data,BUFSIZ);
+            if (rc <= 0)
+                return -1;
+            s->buffer.size = rc;
+            s->buffer.pos = 0;
+            left = rc;
+        }
+        left = left > req ? req : left;
+        s_append(s,s->buffer.data + s->buffer.pos,left);
+        s->buffer.pos += left;
+    };
+    s->pos = pos;
+    return 1;
 }
 
 static inline void *s_get_p_at_pos(sereal_t *s, u32 pos,u32 req) {
-        if (pos + req > s->size) 
-                rb_raise(rb_eRangeError,"position is out of bounds (%d + %d >  %d)",pos,req,s->size);
-        return &s->data[pos];
+    // returning s->data[pos], so we just make size count from 0
+    if (pos + req >= s->size) {
+        if (s->flags & FLAG_STREAM) {
+            if (s_read_stream(s,pos + req + 1) < 0) {
+                s_raise(s,rb_eRangeError,"stream request for %d bytes failed (err: %s)",
+                        req,strerror(errno));
+            }
+        } else {
+            u32 size = s->size;
+            s_raise(s,rb_eRangeError,"position is out of bounds (%d + %d >  %d)",pos,req,size);
+        }
+    }
+    return &s->data[pos];
 }
 
 static inline void *s_get_p_at_pos_bang(sereal_t *s, u32 pos,u32 req) {
-        void *p = s_get_p_at_pos(s,pos,req);
-        s->pos += req;
-        return p;
+    void *p = s_get_p_at_pos(s,pos,req);
+    s->pos += req;
+    return p;
 }
 
+static inline void *s_end_p(sereal_t *s) {
+    return s_get_p_at_pos(s,s->size - 1,0);
+}
+static inline void *s_get_p_req_inclusive(sereal_t *s, int req) {
+    return s_get_p_at_pos(s,s->pos,req > 0 ? req - 1 : 0);
+}
+static inline void *s_get_p_req(sereal_t *s, int req) {
+    return s_get_p_at_pos(s,s->pos,req);
+}
 static inline void *s_get_p(sereal_t *s) {
-        return s_get_p_at_pos(s,s->pos,0);
+    return s_get_p_at_pos(s,s->pos,0);
 }
 
 static inline u8 s_get_u8(sereal_t *s) {
-        return *((u8 *) s_get_p(s));
+    return *((u8 *) s_get_p(s));
 }
 
 static inline u8 s_get_u8_bang(sereal_t *s) {
-        u8 r = s_get_u8(s);
-        s->pos++;
-        return r;
+    u8 r = s_get_u8(s);
+    s->pos++;
+    return r;
 }
 
 static inline u32 s_get_u32_bang(sereal_t *s) {
-        u32 *r = (u32 *) s_get_p_at_pos(s,s->pos,sizeof(u32) - 1); /* current position + 3 bytes */
-        s->pos += sizeof(u32);
-        return *r;
+    u32 *r = (u32 *) s_get_p_at_pos(s,s->pos,sizeof(u32) - 1); /* current position + 3 bytes */
+    s->pos += sizeof(*r);
+    return *r;
+}
+
+static inline float s_get_float_bang(sereal_t *s) {
+    float *f = (float *) s_get_p_at_pos(s,s->pos,sizeof(*f) - 1);
+    s->pos += sizeof(*f);
+    return *f;
+}
+
+static inline double s_get_double_bang(sereal_t *s) {
+    double *d = (double *) s_get_p_at_pos(s,s->pos,sizeof(*d) - 1);
+    s->pos += sizeof(*d);
+    return *d;
+}
+
+static inline long double s_get_long_double_bang(sereal_t *s) {
+    long double *d = (long double *) s_get_p_at_pos(s,s->pos,sizeof(*d) - 1);
+    s->pos += sizeof(*d);
+    return *d;
 }
 
 static inline u32 s_shift_position_bang(sereal_t *s, u32 len) {
-        s->pos += len;
-        return len;
+    s->pos += len;
+    return len;
 }
 
 static void b_dump(u8 *p, u32 len, u32 pos) {
-        int i;
-
-        fprintf(stderr,"\n-----------\n");
-        for (i = 0; i < len; i++) {
-                if (i == pos) 
-                        fprintf(stderr," [%c %d] ",p[i],p[i]);
-                else
-                        fprintf(stderr," (%c %d) ",p[i],p[i]);
-        }
-        fprintf(stderr,"\n-----------\n");
+    int i;
+
+    fprintf(stderr,"\n-----------\n");
+    for (i = 0; i < len; i++) {
+        if (i == pos) 
+            fprintf(stderr," [%c %d] ",p[i],p[i]);
+        else
+            fprintf(stderr," (%c %d) ",p[i],p[i]);
+    }
+    fprintf(stderr,"\n-----------\n");
 }
 
 static void s_dump(sereal_t *s) {
-        b_dump(s->data,s->size,s->pos);
+    E("[ pos: %d, size: %d, rsize: %d ]\n",s->pos,s->size,s->rsize);
+    b_dump(s->data,s->size,s->pos);
 }
-

data/ext/sereal/decode.c

@@ -4,295 +4,332 @@
 #include "snappy/csnappy_decompress.c"
 
 static VALUE s_default_reader(sereal_t *s, u8 tag) {
-        // s_dump(s);
-        s_raise(s,rb_eTypeError,"unsupported tag %d [ 0x%x ]",tag,tag);
-        return Qnil;
+    s_raise(s,rb_eTypeError,"unsupported tag %d [ 0x%x ]",tag,tag);
+    return Qnil;
 }
 
 /* VARINT */
 static u64 s_get_varint_bang(sereal_t *s) {
-        u64 uv = 0;
-        unsigned int lshift = 0;
-        while (s_get_u8(s) & 0x80) {
-                uv |= ((u64)(s_get_u8_bang(s) & 0x7F) << lshift);
-                lshift += 7;
-                if (lshift > (sizeof(uv) * 8))
-                        s_raise(s,rb_eTypeError, "varint too big");
-        }
-        uv |= ((u64)(s_get_u8_bang(s)) << lshift);
-        return uv;
+    u64 uv = 0;
+    unsigned int lshift = 0;
+    while (s_get_u8(s) & 0x80) {
+        uv |= ((u64)(s_get_u8_bang(s) & 0x7F) << lshift);
+        lshift += 7;
+        if (lshift > (sizeof(uv) * 8))
+            s_raise(s,rb_eTypeError, "varint too big");
+    }
+    uv |= ((u64)(s_get_u8_bang(s)) << lshift);
+    return uv;
 }
 
 /* ZIGZAG */
 static VALUE s_read_zigzag(sereal_t *s, u8 tag) {
-        signed long long z = 0;
-        u64 v = s_get_varint_bang(s);
-        if (v & 1) {
-                z = ((long) v >> 1);
-        } else {
-                z = (long) v >> 1;
-        }
-        return LL2NUM(z);
+    signed long long z = 0;
+    u64 v = s_get_varint_bang(s);
+    if (v & 1) {
+        z = ((long) v >> 1);
+    } else {
+        z = (long) v >> 1;
+    }
+    return LL2NUM(z);
 }
 
 /* VARINT */
 static VALUE s_read_varint(sereal_t *s, u8 tag) {
-        return ULL2NUM(s_get_varint_bang(s));
+    return ULL2NUM(s_get_varint_bang(s));
 }
 
 /* FLOAT */
 static VALUE s_read_float(sereal_t *s, u8 tag) {
-        float *f = (float *) s_get_p_at_pos_bang(s,s->pos,sizeof(*f));
-        return DBL2NUM((double) *f);
+    return DBL2NUM(s_get_float_bang(s));
 }
 
 /* DOUBLE */
 static VALUE s_read_double(sereal_t *s, u8 tag) {
-        double *d = (double *) s_get_p_at_pos_bang(s,s->pos,sizeof(*d));
-        return DBL2NUM(*d);
+    return DBL2NUM(s_get_double_bang(s));
 }
 
 /* LONG DOUBLE */
 static VALUE s_read_long_double(sereal_t *s, u8 tag) {
-        long double *d = (long double *) s_get_p_at_pos_bang(s,s->pos,sizeof(*d));
-        return DBL2NUM((double) *d);
+    return DBL2NUM((double) s_get_long_double(s));
 }
 
 /* POS */
 static VALUE s_read_small_positive_int(sereal_t *s, u8 tag) {
-        return INT2FIX(tag);
+    return INT2FIX(tag);
 }
 
 /* NEG */
 static VALUE s_read_small_negative_int(sereal_t *s, u8 tag) {
-        return INT2FIX(tag - 32);
+    return INT2FIX(tag - 32);
 }
 
 /* ARRAY */
 static inline VALUE s_read_array_with_len(sereal_t *s, u32 len) {
-        VALUE arr[len];
-        register u32 i;
-        for (i = 0; i < len; i++)
-                arr[i] = sereal_to_rb_object(s);
-        return rb_ary_new4(len,arr);
+    VALUE arr[len];
+    register u32 i;
+    for (i = 0; i < len; i++)
+        arr[i] = sereal_to_rb_object(s);
+    return rb_ary_new4(len,arr);
 }
 
 /* ARRAY */ 
 static VALUE s_read_array(sereal_t *s,u8 tag) {
-        return s_read_array_with_len(s,s_get_varint_bang(s));
+    return s_read_array_with_len(s,s_get_varint_bang(s));
 }
 
 /* ARRAY */ 
 static VALUE s_read_arrayref(sereal_t *s, u8 tag) {
-        return s_read_array_with_len(s,tag & SRL_MASK_ARRAYREF_COUNT);       
+    return s_read_array_with_len(s,tag & SRL_MASK_ARRAYREF_COUNT);       
 }
 
 /* HASH */ 
 static inline VALUE s_read_hash_with_len(sereal_t *s, u32 len) {
-        VALUE hash = rb_hash_new();
-        register u32 i;
-        for (i = 0; i < len; i++) {
-                VALUE key = sereal_to_rb_object(s);
-                VALUE value = sereal_to_rb_object(s);
-                rb_hash_aset(hash,key,value);
-        }
-        return hash;
+    VALUE hash = rb_hash_new();
+    register u32 i;
+    for (i = 0; i < len; i++) {
+        VALUE key = sereal_to_rb_object(s);
+        VALUE value = sereal_to_rb_object(s);
+        rb_hash_aset(hash,key,value);
+    }
+    return hash;
 }
 
 /* HASH */ 
 static VALUE s_read_hash(sereal_t *s, u8 tag) {
-        return s_read_hash_with_len(s,s_get_varint_bang(s));
+    return s_read_hash_with_len(s,s_get_varint_bang(s));
 }
 
 /* HASH */ 
 static VALUE s_read_hashref(sereal_t *s, u8 tag) {
-        return s_read_hash_with_len(s,tag  & SRL_MASK_HASHREF_COUNT);
+    return s_read_hash_with_len(s,tag  & SRL_MASK_HASHREF_COUNT);
 }
 
 static VALUE s_read_rb_string_bang(sereal_t *s,u8 t) {
-        u32 len = 0;
-        VALUE string;
-#define RETURN_STRING(fx_l,fx_gen)              \
-        do {                                    \
-                len = fx_l;                     \
-                string = fx_gen;                \
-                s_shift_position_bang(s,len);   \
-                return string;                  \
-        } while(0);
-
-        if (t == SRL_HDR_STR_UTF8) {
-                RETURN_STRING(s_get_varint_bang(s),
-                              rb_enc_str_new(s_get_p(s),len,
-                              rb_utf8_encoding()));
-        } else if (t == SRL_HDR_BINARY || t == SRL_HDR_SYM) {
-                RETURN_STRING(s_get_varint_bang(s),
-                              rb_str_new(s_get_p(s),len));
-        } else if (IS_SHORT_BINARY(t)) {
-                RETURN_STRING((t & SRL_MASK_SHORT_BINARY_LEN),
-                              rb_str_new(s_get_p(s), len));
-        }
+    u32 len = 0;
+    VALUE string;
+// len - 1: we also use the current byte, similar to u32,float.. casts
+#define RETURN_STRING(fx_l,fx_gen)                             \
+    do {                                                       \
+        len = fx_l;                                            \
+        u8 *ptr = len == 0 ? 0 : s_get_p_req_inclusive(s,len); \
+        string = fx_gen;                                       \
+        s_shift_position_bang(s,len);                          \
+        return string;                                         \
+    } while(0);
+
+    if (t == SRL_HDR_STR_UTF8) {
+        RETURN_STRING(s_get_varint_bang(s),
+                  rb_enc_str_new(ptr,len,
+                  rb_utf8_encoding()));
+    } else if (t == SRL_HDR_BINARY || t == SRL_HDR_SYM) {
+        RETURN_STRING(s_get_varint_bang(s),
+                  rb_str_new(ptr,len));
+    } else if (IS_SHORT_BINARY(t)) {
+        RETURN_STRING((t & SRL_MASK_SHORT_BINARY_LEN),
+                  rb_str_new(ptr, len));
+    }
 #undef RETURN_STRING
-        s_raise(s,rb_eTypeError, "undefined string type %d",t);
+    s_raise(s,rb_eTypeError, "undefined string type %d",t);
 }
 
 static VALUE s_read_next_rb_string_bang(sereal_t *s) {
-        return s_read_rb_string_bang(s,s_get_u8_bang(s));
+    return s_read_rb_string_bang(s,s_get_u8_bang(s));
 }
 
 static VALUE s_read_regexp(sereal_t *s, u8 tag) {
-        VALUE pattern = s_read_next_rb_string_bang(s);
-        VALUE modifiers = s_read_next_rb_string_bang(s);
-
-        u32 flags = 0;
-        if (strchr(RSTRING_PTR(modifiers),'i')) 
-                flags |= IGNORECASE;       
-        
-        if (strchr(RSTRING_PTR(modifiers),'m')) 
-                flags |= MULTILINE;       
-
-        if (strchr(RSTRING_PTR(modifiers),'x')) 
-                flags |= EXTENDED;
-	#ifdef RUBINIUS
-		return rb_reg_new(RSTRING_PTR(pattern),RSTRING_LEN(pattern),flags);
-	#else
-		return rb_reg_new_str(pattern,flags);
-	#endif
+    VALUE pattern = s_read_next_rb_string_bang(s);
+    VALUE modifiers = s_read_next_rb_string_bang(s);
+
+    u32 flags = 0;
+    if (strchr(RSTRING_PTR(modifiers),'i'))
+        flags |= IGNORECASE;
+
+    if (strchr(RSTRING_PTR(modifiers),'m'))
+        flags |= MULTILINE;
+
+    if (strchr(RSTRING_PTR(modifiers),'x'))
+        flags |= EXTENDED;
+
+#ifdef RUBINIUS
+    return rb_reg_new(RSTRING_PTR(pattern),RSTRING_LEN(pattern),flags);
+#else
+    return rb_reg_new_str(pattern,flags);
+#endif
 }
 
 static VALUE s_read_nil(sereal_t *s, u8 tag) {
-        return Qnil;
+    return Qnil;
 }
 
 static VALUE s_read_true(sereal_t *s, u8 tag) {
-        return Qtrue;
+    return Qtrue;
 }
 
 static VALUE s_read_false(sereal_t *s, u8 tag) {
-        return Qfalse;
+    return Qfalse;
 }
 
 static VALUE s_read_pad(sereal_t *s, u8 tag) {
-        /* just skip this byte and go forward */
-        return sereal_to_rb_object(s);
+    /* just skip this byte and go forward */
+    return sereal_to_rb_object(s);
 }
 
 static VALUE s_read_extend(sereal_t *s, u8 tag) {
-        s_raise(s,rb_eArgError,"extend tags are not supported");
+    s_raise(s,rb_eArgError,"extend tags are not supported");
 }
 
 static VALUE s_read_ref(sereal_t *s, u8 tag) {
-        u64 off = s_get_varint_bang(s);
-        if (s->tracked == Qnil)
-                s_raise(s,rb_eArgError,"there are no references stored");
-        return rb_hash_aref(s->tracked,INT2FIX(off + s->hdr_end));
+    u64 off = s_get_varint_bang(s);
+    if (s->tracked == Qnil)
+        s_raise(s,rb_eArgError,"there are no references stored");
+    return rb_hash_aref(s->tracked,INT2FIX(off + s->hdr_end));
 }
 
 static VALUE s_read_copy(sereal_t *s, u8 tag) {
-        VALUE ref = s_red_ref(s,tag);
-        return rb_obj_dup(ref);
+    VALUE ref = s_red_ref(s,tag);
+    return rb_obj_dup(ref);
 }
 
 VALUE sereal_to_rb_object(sereal_t *s) {
-        u8 t, tracked;
-        S_RECURSE_INC(s);
-        u32 pos;
-        while (s->pos < s->size) {
-                t = s_get_u8_bang(s);
-                tracked = (t & SRL_HDR_TRACK_FLAG ? 1 : 0);
-                t &= ~SRL_HDR_TRACK_FLAG;
-                pos = s->pos;
-
-                S_RECURSE_DEC(s);
-
-                VALUE decoded = (*READERS[t])(s,t);
-                if (tracked) {
-                        if (s->tracked == Qnil) {
-                                s->tracked = rb_hash_new();
-                                rb_gc_register_address(&s->tracked);
-                        }
-                        rb_hash_aset(s->tracked,INT2FIX(pos),decoded);
-                }
-                return decoded;
+    u8 t, tracked;
+    S_RECURSE_INC(s);
+    u32 pos;
+    while (s->pos < s->size || (s->flags & FLAG_STREAM)) {
+        t = s_get_u8_bang(s);
+        tracked = (t & SRL_HDR_TRACK_FLAG ? 1 : 0);
+        t &= ~SRL_HDR_TRACK_FLAG;
+        pos = s->pos;
+
+        S_RECURSE_DEC(s);
+
+        VALUE decoded = (*READERS[t])(s,t);
+        if (tracked) {
+            if (s->tracked == Qnil) {
+                s->tracked = rb_hash_new();
+                rb_gc_register_address(&s->tracked);
+            }
+            rb_hash_aset(s->tracked,INT2FIX(pos),decoded);
         }
-        s_raise(s,rb_eArgError,"bad packet, or broken decoder");
-        return Qnil;
+        return decoded;
+    }
+    s_raise(s,rb_eArgError,"bad packet, or broken decoder");
+    return Qnil;
 }
 
 VALUE method_sereal_decode(VALUE self, VALUE args) {
-        u32 argc = RARRAY_LEN(args);
-        if (argc < 1)
-                rb_raise(rb_eArgError,"need at least 1 argument (object)");
-        VALUE payload = rb_ary_shift(args);
-        if (TYPE(payload) != T_STRING) 
-                rb_raise(rb_eTypeError,"can not decode objects of type %s",rb_obj_classname(payload));
-        u8 have_block = rb_block_given_p();
-        sereal_t *s = s_create();
-        u64 offset = 0;
+    u32 argc = RARRAY_LEN(args);
+    if (argc < 1)
+        rb_raise(rb_eArgError,"need at least 1 argument (object)");
+    VALUE payload = rb_ary_shift(args);
+    u8 have_block = rb_block_given_p();
+    sereal_t *s = s_create();
+    u64 offset = 0;
+    if (TYPE(payload) == T_FILE) {
+        if (!have_block)
+            s_raise(s,rb_eTypeError,"block is required when reading from a stream")
+
+        rb_io_t *fptr;
+        GetOpenFile(payload, fptr);
+        s->flags |= FLAG_STREAM;
+        s->fd = fptr->fd;
+    } else if (TYPE(payload) != T_STRING) {
+        rb_raise(rb_eTypeError,"can not decode objects of type %s",rb_obj_classname(payload));
+    }
+
 again: 
-        if (offset >= RSTRING_LEN(payload)) {
-            free(s);
+    s->pos = 0;
+    s_reset_tracker(s);
+
+    if (s->flags & FLAG_STREAM) {
+        s->size = 0;
+        s->rsize = 0;
+        if (s_read_stream(s,__MIN_SIZE) < 0) {
+            s_destroy(s);
             return Qnil;
         }
+
+    } else {
+        u32 size = RSTRING_LEN(payload) - offset;
+        if (offset > RSTRING_LEN(payload) || (offset > 0 && size < __MIN_SIZE)) {
+            s_destroy(s);
+            return Qnil;
+        }
+        if (size < __MIN_SIZE)
+            s_raise(s,rb_eTypeError,"size(%d) is less then min packet size %d, offset: %d",size,__MIN_SIZE,offset);
+
         s->flags |= FLAG_NOT_MINE;
         s->data = RSTRING_PTR(payload) + offset;
-        s->size = RSTRING_LEN(payload) - offset;
-        s->pos = 0;
-        s->tracked = Qnil;
-        if (s->size < 6 || s_get_u32_bang(s) != SRL_MAGIC_STRING_LILIPUTIAN) 
-                s_raise(s,rb_eTypeError,"invalid header"); 
-
-        u8 version = s_get_u8_bang(s);
-        u8 suffix = s_get_varint_bang(s);
-        u8 is_compressed;
-
-        if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY)
-                is_compressed = __SNAPPY;
-        else if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY_INCR)
-                is_compressed = __SNAPPY_INCR;
-        else
-                is_compressed = __RAW;
-        if (is_compressed) {
-                u32 uncompressed_len;
-                u32 compressed_len;
-
-                if (is_compressed == __SNAPPY_INCR) {
-                        compressed_len = s_get_varint_bang(s);
-                } else {
-                        compressed_len = s->size - s->pos;
-                }
-
-                int snappy_header_len = csnappy_get_uncompressed_length(s_get_p(s),
-                                                                        compressed_len,
-                                                                        &uncompressed_len);
-                if (snappy_header_len == CSNAPPY_E_HEADER_BAD) 
-                    rb_raise(rb_eTypeError,"invalid snappy header");
-                u8 *uncompressed = alloc_or_raise(uncompressed_len);
-                int done = csnappy_decompress(s_get_p(s),
-                                              compressed_len,
-                                              uncompressed,
-                                              uncompressed_len) == CSNAPPY_E_OK ? 1 : 0;
-                if (!done)
-                        s_raise(s,rb_eTypeError, "decompression failed error: %d type: %d, unompressed size: %d compressed size: %d",done,is_compressed,uncompressed_len,compressed_len);
-
-                offset += s->pos + compressed_len;
-                s->data = uncompressed;
-                s->size = uncompressed_len;
-                s->pos = 0;
-                s->flags &= ~FLAG_NOT_MINE;
+        s->size = size;
+    }
+
+    u32 magic = s_get_u32_bang(s);
+    if (magic != SRL_MAGIC_STRING_LILIPUTIAN)
+        s_raise(s,rb_eTypeError,"invalid header: %d (%x)",magic,magic);
+
+    u8 version = s_get_u8_bang(s);
+    u8 suffix = s_get_varint_bang(s);
+    u8 is_compressed;
+
+    if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY)
+        is_compressed = __SNAPPY;
+    else if ((version & SRL_PROTOCOL_ENCODING_MASK) == SRL_PROTOCOL_ENCODING_SNAPPY_INCR)
+        is_compressed = __SNAPPY_INCR;
+    else
+        is_compressed = __RAW;
+
+    if (is_compressed) {
+        u32 uncompressed_len;
+        u32 compressed_len;
+
+        if (is_compressed == __SNAPPY_INCR) {
+            compressed_len = s_get_varint_bang(s);
+        } else {
+            if (s->flags & FLAG_STREAM)
+                s_raise(s,rb_eTypeError,"parsing non incremental compressed objects, from stream of data, is not supported");
+
+            compressed_len = s->size - s->pos;
         }
-        s->hdr_end = s->pos;
-        VALUE result = sereal_to_rb_object(s);
-        if (is_compressed && have_block) {
-                free(s->data);
-                s->data = NULL;
-                rb_yield(result);
-                goto again;
+
+        if (s->flags & FLAG_STREAM)
+            s_get_p_req_inclusive(s,compressed_len);
+
+        int snappy_header_len = csnappy_get_uncompressed_length(s_get_p_req_inclusive(s,compressed_len),
+                                                                compressed_len,
+                                                                &uncompressed_len);
+        if (snappy_header_len == CSNAPPY_E_HEADER_BAD)
+            s_raise(s,rb_eTypeError,"invalid snappy header");
+
+        u8 *uncompressed = s_alloc_or_raise(s,uncompressed_len);
+        int done = csnappy_decompress(s_get_p_req_inclusive(s,compressed_len),
+                                      compressed_len,
+                                      uncompressed,
+                                      uncompressed_len) == CSNAPPY_E_OK ? 1 : 0;
+        if (!done)
+            s_raise(s,rb_eTypeError, "decompression failed error: %d type: %d, unompressed size: %d compressed size: %d",
+                                      done,is_compressed,uncompressed_len,compressed_len);
+
+        s_free_data_if_not_mine(s);
+        s->data = uncompressed;
+        s->size = uncompressed_len;
+        offset += s->pos + compressed_len;
+        s->pos = 0;
+        s->flags &= ~FLAG_NOT_MINE;
+    }
+    s->hdr_end = s->pos;
+    VALUE result = sereal_to_rb_object(s);
+    if (!is_compressed)
+        offset += s->pos;
+    if (have_block) {
+        rb_yield(result);
+        s_free_data_if_not_mine(s);
+        goto again;
+    } else {
+        if (s->pos < s->size) {
+            s_raise(s,rb_eTypeError,"there is still some data left in the buffer, use block read, otherwise we are losing it");
         }
         s_destroy(s);
-        if (have_block) {
-            rb_yield(result);
-            return Qnil;
-        }
         return result;
+    }
 }