sereal 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/ext/sereal/buffer.h +89 -0
- data/ext/sereal/decode.c +238 -0
- data/ext/sereal/decode.h +282 -0
- data/ext/sereal/encode.c +269 -0
- data/ext/sereal/encode.h +1 -0
- data/ext/sereal/extconf.rb +8 -0
- data/ext/sereal/proto.h +73 -0
- data/ext/sereal/sereal.c +12 -0
- data/ext/sereal/sereal.h +73 -0
- data/ext/sereal/snappy/csnappy.h +129 -0
- data/ext/sereal/snappy/csnappy_compress.c +659 -0
- data/ext/sereal/snappy/csnappy_decompress.c +414 -0
- data/ext/sereal/snappy/csnappy_internal.h +147 -0
- data/ext/sereal/snappy/csnappy_internal_userspace.h +301 -0
- metadata +75 -0
data/ext/sereal/encode.c
ADDED
@@ -0,0 +1,269 @@
|
|
1
|
+
#include "sereal.h"
|
2
|
+
#include "buffer.h"
|
3
|
+
#include "encode.h"
|
4
|
+
#include "snappy/csnappy_compress.c"
|
5
|
+
|
6
|
+
#define W_SIZE 32
|
7
|
+
#if T_FIXNUM > W_SIZE
|
8
|
+
#define W_SIZE T_FIXNUM
|
9
|
+
#elif T_BIGNUM > W_SIZE
|
10
|
+
#define W_SIZE T_BIGNUM
|
11
|
+
#elif T_FLOAT > W_SIZE
|
12
|
+
#define W_SIZE T_FLOAT
|
13
|
+
#elif T_OBJECT > W_SIZE
|
14
|
+
#define W_SIZE T_OBJECT
|
15
|
+
#elif T_REGEXP > W_SIZE
|
16
|
+
#define W_SIZE T_REGEXP
|
17
|
+
#elif T_STRING > W_SIZE
|
18
|
+
#define W_SIZE T_STRING
|
19
|
+
#elif T_ARRAY > W_SIZE
|
20
|
+
#define W_SIZE T_ARRAY
|
21
|
+
#elif T_HASH > W_SIZE
|
22
|
+
#define W_SIZE T_HASH
|
23
|
+
#elif T_SYMBOL > W_SIZE
|
24
|
+
#define W_SIZE T_SYMBOL
|
25
|
+
#elif T_TRUE > W_SIZE
|
26
|
+
#define W_SIZE T_TRUE
|
27
|
+
#elif T_FALSE > W_SIZE
|
28
|
+
#define W_SIZE T_FALSE
|
29
|
+
#elif T_NIL > W_SIZE
|
30
|
+
#define W_SIZE T_NIL
|
31
|
+
#endif
|
32
|
+
|
33
|
+
/* function pointer array */
|
34
|
+
void (*WRITER[W_SIZE])(sereal_t *,VALUE);
|
35
|
+
|
36
|
+
static void rb_object_to_sereal(sereal_t *s, VALUE object);
|
37
|
+
static void s_append_varint(sereal_t *s,u64 n);
|
38
|
+
static void s_append_hdr_with_varint(sereal_t *s,u8 hdr, u64 n);
|
39
|
+
static void s_append_zigzag(sereal_t *s,u64 n);
|
40
|
+
static void s_append_string(sereal_t *s,u8 *string, u32 len,u8 is_utf8);
|
41
|
+
static void s_append_rb_string(sereal_t *s, VALUE object);
|
42
|
+
static void s_append_array(sereal_t *s, VALUE object);
|
43
|
+
static void s_append_hash(sereal_t *s, VALUE object);
|
44
|
+
static void s_append_symbol(sereal_t *s, VALUE object);
|
45
|
+
static void s_append_object(sereal_t *s, VALUE object);
|
46
|
+
static void s_append_regexp(sereal_t *s, VALUE object);
|
47
|
+
static void s_append_integer(sereal_t *s, VALUE object);
|
48
|
+
static void s_append_double(sereal_t *s, VALUE object);
|
49
|
+
static void s_append_true(sereal_t *s, VALUE object);
|
50
|
+
static void s_append_false(sereal_t *s, VALUE object);
|
51
|
+
static void s_append_nil(sereal_t *s, VALUE object);
|
52
|
+
static void s_default_writer(sereal_t *s, VALUE object);
|
53
|
+
|
54
|
+
void s_init_writers(void) {
|
55
|
+
u32 i;
|
56
|
+
for (i = 0; i < sizeof(WRITER)/sizeof(WRITER[0]); i++)
|
57
|
+
WRITER[i] = s_default_writer;
|
58
|
+
WRITER[T_FIXNUM] = s_append_integer;
|
59
|
+
WRITER[T_BIGNUM] = s_append_integer;
|
60
|
+
WRITER[T_FLOAT] = s_append_double;
|
61
|
+
WRITER[T_OBJECT] = s_append_object;
|
62
|
+
WRITER[T_REGEXP] = s_append_regexp;
|
63
|
+
WRITER[T_STRING] = s_append_rb_string;
|
64
|
+
WRITER[T_ARRAY] = s_append_array;
|
65
|
+
WRITER[T_HASH] = s_append_hash;
|
66
|
+
WRITER[T_SYMBOL] = s_append_symbol;
|
67
|
+
WRITER[T_TRUE] = s_append_true;
|
68
|
+
WRITER[T_FALSE] = s_append_false;
|
69
|
+
WRITER[T_NIL] = s_append_nil;
|
70
|
+
}
|
71
|
+
|
72
|
+
static void s_default_writer(sereal_t *s, VALUE object) {
|
73
|
+
rb_raise(rb_eTypeError, "invalid type for input %s",rb_obj_classname(object));
|
74
|
+
}
|
75
|
+
|
76
|
+
|
77
|
+
static inline void s_append_varint(sereal_t *s,u64 n) {
|
78
|
+
while (n >= 0x80) {
|
79
|
+
s_append_u8(s,((n & 0x7f) | 0x80));
|
80
|
+
n >>= 7;
|
81
|
+
}
|
82
|
+
s_append_u8(s,n);
|
83
|
+
}
|
84
|
+
|
85
|
+
static inline void s_append_hdr_with_varint(sereal_t *s,u8 hdr, u64 n) {
|
86
|
+
s_append_u8(s,hdr);
|
87
|
+
s_append_varint(s,n);
|
88
|
+
}
|
89
|
+
|
90
|
+
static inline void s_append_zigzag(sereal_t *s,u64 n) {
|
91
|
+
s_append_hdr_with_varint(s,SRL_HDR_ZIGZAG,
|
92
|
+
(n << 1) ^ (n >> (sizeof(long) * 8 - 1)));
|
93
|
+
}
|
94
|
+
|
95
|
+
static inline void s_append_string(sereal_t *s,u8 *string, u32 len,u8 is_utf8) {
|
96
|
+
if (is_utf8) {
|
97
|
+
s_append_hdr_with_varint(s,SRL_HDR_STR_UTF8,len);
|
98
|
+
} else {
|
99
|
+
if (len < SRL_MASK_SHORT_BINARY_LEN) {
|
100
|
+
s_append_u8(s,SRL_HDR_SHORT_BINARY_LOW | (u8)len);
|
101
|
+
} else {
|
102
|
+
s_append_hdr_with_varint(s,SRL_HDR_BINARY,len);
|
103
|
+
}
|
104
|
+
}
|
105
|
+
s_append(s,string,len);
|
106
|
+
}
|
107
|
+
|
108
|
+
static void s_append_rb_string(sereal_t *s, VALUE object) {
|
109
|
+
s_append_string(s,RSTRING_PTR(object),
|
110
|
+
RSTRING_LEN(object),
|
111
|
+
(is_ascii_string(object) ? FALSE : TRUE));
|
112
|
+
}
|
113
|
+
|
114
|
+
#define REF_THRESH(thresh,low,high) \
|
115
|
+
do { \
|
116
|
+
if (len < (thresh)) \
|
117
|
+
s_append_u8(s, low | (u8) len); \
|
118
|
+
else \
|
119
|
+
s_append_hdr_with_varint(s,high,len); \
|
120
|
+
} while(0);
|
121
|
+
static void s_append_array(sereal_t *s, VALUE object) {
|
122
|
+
u32 i,len = RARRAY_LEN(object);
|
123
|
+
REF_THRESH(SRL_MASK_ARRAYREF_COUNT,SRL_HDR_ARRAYREF,SRL_HDR_ARRAY);
|
124
|
+
|
125
|
+
for (i = 0; i < len; i++)
|
126
|
+
rb_object_to_sereal(s,rb_ary_entry(object,i));
|
127
|
+
}
|
128
|
+
|
129
|
+
|
130
|
+
int s_hash_foreach(VALUE key, VALUE value, VALUE sereal_t_object) {
|
131
|
+
if (key == Qundef)
|
132
|
+
return ST_CONTINUE;
|
133
|
+
rb_object_to_sereal((sereal_t *) sereal_t_object,key);
|
134
|
+
rb_object_to_sereal((sereal_t *) sereal_t_object,value);
|
135
|
+
return ST_CONTINUE;
|
136
|
+
}
|
137
|
+
static void s_append_hash(sereal_t *s, VALUE object) {
|
138
|
+
u32 len = RHASH_SIZE(object);
|
139
|
+
REF_THRESH(SRL_MASK_HASHREF_COUNT,SRL_HDR_HASHREF,SRL_HDR_HASH);
|
140
|
+
rb_hash_foreach(object, s_hash_foreach, (VALUE) s);
|
141
|
+
}
|
142
|
+
#undef REF_THRESH
|
143
|
+
|
144
|
+
/*
|
145
|
+
convert symbols to strings
|
146
|
+
*/
|
147
|
+
static void s_append_symbol(sereal_t *s, VALUE object) {
|
148
|
+
VALUE string = rb_sym_to_s(object);
|
149
|
+
s_append_rb_string(s,string);
|
150
|
+
}
|
151
|
+
|
152
|
+
/*
|
153
|
+
call object.to_srl and serialize the result
|
154
|
+
*/
|
155
|
+
static void s_append_object(sereal_t *s, VALUE object) {
|
156
|
+
rb_object_to_sereal(s,rb_funcall(object,rb_intern("to_srl"),0));
|
157
|
+
}
|
158
|
+
|
159
|
+
|
160
|
+
// <PATTERN-STR-TAG> <MODIFIERS-STR-TAG>
|
161
|
+
static void s_append_regexp(sereal_t *s, VALUE object) {
|
162
|
+
s_append_u8(s,SRL_HDR_REGEXP);
|
163
|
+
rb_encoding *enc = rb_enc_get(object);
|
164
|
+
VALUE pattern;
|
165
|
+
#ifndef RREGEXP_SRC_PTR
|
166
|
+
VALUE string = RREGEXP_SRC(object);
|
167
|
+
pattern = rb_enc_str_new(RSTRING_PTR(string),RSTRING_LEN(string),enc);
|
168
|
+
#else
|
169
|
+
pattern = rb_enc_str_new(RREGEXP_SRC_PTR(object),RREGEXP_SRC_LEN(object), enc);
|
170
|
+
#endif
|
171
|
+
s_append_rb_string(s,pattern);
|
172
|
+
|
173
|
+
int flags = rb_reg_options(object);
|
174
|
+
VALUE f = rb_str_new("",0);
|
175
|
+
if (flags & IGNORECASE)
|
176
|
+
rb_str_cat(f,"i",1);
|
177
|
+
if (flags & EXTENDED)
|
178
|
+
rb_str_cat(f,"x",1);
|
179
|
+
if (flags & MULTILINE)
|
180
|
+
rb_str_cat(f,"m",1);
|
181
|
+
s_append_rb_string(s,f);
|
182
|
+
}
|
183
|
+
|
184
|
+
|
185
|
+
static void s_append_integer(sereal_t *s, VALUE object) {
|
186
|
+
long long v = FIXNUM_P(object) ? FIX2LONG(object) : rb_num2ll(object);
|
187
|
+
if (v >= 0) {
|
188
|
+
if (v < 16)
|
189
|
+
s_append_u8(s,SRL_HDR_POS_LOW | (u8) v);
|
190
|
+
else {
|
191
|
+
if (!FIXNUM_P(object))
|
192
|
+
s_append_hdr_with_varint(s,SRL_HDR_VARINT,NUM2ULL(object));
|
193
|
+
else
|
194
|
+
s_append_hdr_with_varint(s,SRL_HDR_VARINT,v);
|
195
|
+
}
|
196
|
+
} else {
|
197
|
+
if (v > -17)
|
198
|
+
s_append_u8(s,SRL_HDR_NEG_LOW | ((u8) v + 32));
|
199
|
+
else
|
200
|
+
s_append_zigzag(s,v);
|
201
|
+
}
|
202
|
+
}
|
203
|
+
static void s_append_double(sereal_t *s, VALUE object) {
|
204
|
+
double d = NUM2DBL(object);
|
205
|
+
s_append_u8(s,SRL_HDR_DOUBLE);
|
206
|
+
s_append(s,&d,sizeof(d));
|
207
|
+
}
|
208
|
+
static void s_append_true(sereal_t *s, VALUE object) {
|
209
|
+
s_append_u8(s,SRL_HDR_TRUE);
|
210
|
+
}
|
211
|
+
static void s_append_false(sereal_t *s, VALUE object) {
|
212
|
+
s_append_u8(s,SRL_HDR_FALSE);
|
213
|
+
}
|
214
|
+
static void s_append_nil(sereal_t *s, VALUE object) {
|
215
|
+
s_append_u8(s,SRL_HDR_UNDEF);
|
216
|
+
}
|
217
|
+
|
218
|
+
/* writer function pointers */
|
219
|
+
static void rb_object_to_sereal(sereal_t *s, VALUE object) {
|
220
|
+
S_RECURSE_INC(s);
|
221
|
+
(*WRITER[TYPE(object)])(s,object);
|
222
|
+
S_RECURSE_DEC(s);
|
223
|
+
}
|
224
|
+
|
225
|
+
VALUE method_sereal_encode(VALUE self, VALUE args) {
|
226
|
+
sereal_t *s = s_create();
|
227
|
+
u32 argc = RARRAY_LEN(args);
|
228
|
+
if (argc < 1)
|
229
|
+
rb_raise(rb_eArgError,"need at least 1 argument (object)");
|
230
|
+
VALUE payload = rb_ary_shift(args);
|
231
|
+
VALUE compress = Qfalse;
|
232
|
+
if (argc == 2)
|
233
|
+
compress = rb_ary_shift(args);
|
234
|
+
int do_compress = (compress == Qtrue ? TRUE : FALSE);
|
235
|
+
|
236
|
+
// setup header
|
237
|
+
s_append_u32(s,SRL_MAGIC_STRING_LILIPUTIAN);
|
238
|
+
s_append_u8(s,SRL_PROTOCOL_VERSION | (do_compress ? SRL_PROTOCOL_ENCODING_SNAPPY : SRL_PROTOCOL_ENCODING_RAW));
|
239
|
+
s_append_u8(s,0x0);
|
240
|
+
u32 s_header_len = s->pos;
|
241
|
+
|
242
|
+
// serialize
|
243
|
+
rb_object_to_sereal(s,payload);
|
244
|
+
|
245
|
+
// compress
|
246
|
+
if (do_compress) {
|
247
|
+
u32 s_body_len = s->size - s_header_len;
|
248
|
+
u32 compressed_len = csnappy_max_compressed_length(s_body_len);
|
249
|
+
|
250
|
+
u8 *working_buf = alloc_or_raise(CSNAPPY_WORKMEM_BYTES);
|
251
|
+
u8 *compressed = alloc_or_raise(compressed_len + s_header_len);
|
252
|
+
|
253
|
+
COPY(s_get_p_at_pos(s,0,0),compressed,s_header_len);
|
254
|
+
|
255
|
+
csnappy_compress(s_get_p_at_pos(s,s_header_len,1),
|
256
|
+
s_body_len,
|
257
|
+
(compressed + s_header_len),
|
258
|
+
&compressed_len,
|
259
|
+
working_buf,
|
260
|
+
CSNAPPY_WORKMEM_BYTES_POWER_OF_TWO);
|
261
|
+
free(s->data);
|
262
|
+
s->data = compressed;
|
263
|
+
s->size = compressed_len + s_header_len;
|
264
|
+
}
|
265
|
+
|
266
|
+
VALUE result = rb_str_new(s->data,s->size);
|
267
|
+
s_destroy(s);
|
268
|
+
return result;
|
269
|
+
}
|
data/ext/sereal/encode.h
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
void s_init_writers(void);
|
data/ext/sereal/proto.h
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
#define SRL_MAGIC_STRING "=srl" /* Magic string for header. Every packet starts with this */
|
2
|
+
#define SRL_MAGIC_STRING_LILIPUTIAN 0x6c72733d /* SRL_MAGIC_STRING as a little endian integer */
|
3
|
+
|
4
|
+
#define SRL_PROTOCOL_VERSION ( 1 ) /* this is the first. for some reason we did not use 0 */
|
5
|
+
#define SRL_PROTOCOL_VERSION_BITS ( 4 ) /* how many bits we use for the version, the rest go to the encoding */
|
6
|
+
#define SRL_PROTOCOL_VERSION_MASK ( ( 1 << SRL_PROTOCOL_VERSION_BITS ) - 1 )
|
7
|
+
|
8
|
+
#define SRL_PROTOCOL_ENCODING_MASK ( ~SRL_PROTOCOL_VERSION_MASK )
|
9
|
+
#define SRL_PROTOCOL_ENCODING_RAW ( 0 << SRL_PROTOCOL_VERSION_BITS )
|
10
|
+
#define SRL_PROTOCOL_ENCODING_SNAPPY ( 1 << SRL_PROTOCOL_VERSION_BITS )
|
11
|
+
|
12
|
+
#define SRL_HDR_POS ((char)0) /* small positive integer - value in low 4 bits (identity) */
|
13
|
+
#define SRL_HDR_POS_LOW ((char)0) /* small positive integer - value in low 4 bits (identity) */
|
14
|
+
#define SRL_HDR_POS_HIGH ((char)15) /* small positive integer - value in low 4 bits (identity) */
|
15
|
+
|
16
|
+
#define SRL_HDR_NEG ((char)16) /* small negative integer - value in low 4 bits (k+32) */
|
17
|
+
#define SRL_HDR_NEG_LOW ((char)16) /* small negative integer - value in low 4 bits (k+32) */
|
18
|
+
#define SRL_HDR_NEG_HIGH ((char)31) /* small negative integer - value in low 4 bits (k+32) */
|
19
|
+
|
20
|
+
#define SRL_HDR_VARINT ((char)32) /* <VARINT> - Varint variable length integer */
|
21
|
+
#define SRL_HDR_ZIGZAG ((char)33) /* <ZIGZAG-VARINT> - Zigzag variable length integer */
|
22
|
+
#define SRL_HDR_FLOAT ((char)34) /* <IEEE-FLOAT> */
|
23
|
+
#define SRL_HDR_DOUBLE ((char)35) /* <IEEE-DOUBLE> */
|
24
|
+
#define SRL_HDR_LONG_DOUBLE ((char)36) /* <IEEE-LONG-DOUBLE> */
|
25
|
+
#define SRL_HDR_UNDEF ((char)37) /* None - Perl undef */
|
26
|
+
#define SRL_HDR_BINARY ((char)38) /* <LEN-VARINT> <BYTES> - binary/(latin1) string */
|
27
|
+
#define SRL_HDR_STR_UTF8 ((char)39) /* <LEN-VARINT> <UTF8> - utf8 string */
|
28
|
+
|
29
|
+
#define SRL_HDR_REFN ((char)40) /* <ITEM-TAG> - ref to next item */
|
30
|
+
#define SRL_HDR_REFP ((char)41) /* <OFFSET-VARINT> - ref to previous item stored at offset */
|
31
|
+
#define SRL_HDR_HASH ((char)42) /* <COUNT-VARINT> [<KEY-TAG> <ITEM-TAG> ...] - count followed by key/value pairs */
|
32
|
+
#define SRL_HDR_ARRAY ((char)43) /* <COUNT-VARINT> [<ITEM-TAG> ...] - count followed by items */
|
33
|
+
#define SRL_HDR_OBJECT ((char)44) /* <STR-TAG> <ITEM-TAG> - class, object-item */
|
34
|
+
#define SRL_HDR_OBJECTV ((char)45) /* <OFFSET-VARINT> <ITEM-TAG> - offset of previously used classname tag - object-item */
|
35
|
+
#define SRL_HDR_ALIAS ((char)46) /* <OFFSET-VARINT> - alias to item defined at offset */
|
36
|
+
#define SRL_HDR_COPY ((char)47) /* <OFFSET-VARINT> - copy of item defined at offset */
|
37
|
+
|
38
|
+
#define SRL_HDR_WEAKEN ((char)48) /* <REF-TAG> - Weaken the following reference */
|
39
|
+
#define SRL_HDR_REGEXP ((char)49) /* <PATTERN-STR-TAG> <MODIFIERS-STR-TAG>*/
|
40
|
+
|
41
|
+
/* Note: Can do reserved check with a range now, but as we start using
|
42
|
+
* them, might have to explicit == check later. */
|
43
|
+
#define SRL_HDR_RESERVED ((char)50) /* reserved */
|
44
|
+
#define SRL_HDR_RESERVED_LOW ((char)50)
|
45
|
+
#define SRL_HDR_RESERVED_HIGH ((char)57)
|
46
|
+
|
47
|
+
#define SRL_HDR_FALSE ((char)58) /* false (PL_sv_no) */
|
48
|
+
#define SRL_HDR_TRUE ((char)59) /* true (PL_sv_yes) */
|
49
|
+
|
50
|
+
#define SRL_HDR_MANY ((char)60) /* <LEN-VARINT> <TYPE-BYTE> <TAG-DATA> - repeated tag (not done yet, will be implemented in version 2) */
|
51
|
+
#define SRL_HDR_PACKET_START ((char)61) /* (first byte of magic string in header) */
|
52
|
+
|
53
|
+
|
54
|
+
#define SRL_HDR_EXTEND ((char)62) /* <BYTE> - for additional tags */
|
55
|
+
#define SRL_HDR_PAD ((char)63) /* (ignored tag, skip to next byte) */
|
56
|
+
#define SRL_HDR_ARRAYREF ((char)64) /* [<ITEM-TAG> ...] - count of items in low 4 bits (ARRAY must be refcnt=1)*/
|
57
|
+
#define SRL_MASK_ARRAYREF_COUNT ((char)15) /* mask to get low bits from tag */
|
58
|
+
#define SRL_HDR_ARRAYREF_LOW ((char)64)
|
59
|
+
#define SRL_HDR_ARRAYREF_HIGH ((char)79)
|
60
|
+
|
61
|
+
|
62
|
+
#define SRL_HDR_HASHREF ((char)80) /* [<KEY-TAG> <ITEM-TAG> ...] - count in low 4 bits, key/value pairs (HASH must be refcnt=1)*/
|
63
|
+
#define SRL_MASK_HASHREF_COUNT ((char)15) /* mask to get low bits from tag */
|
64
|
+
#define SRL_HDR_HASHREF_LOW ((char)80)
|
65
|
+
#define SRL_HDR_HASHREF_HIGH ((char)95)
|
66
|
+
|
67
|
+
#define SRL_HDR_SHORT_BINARY ((char)96) /* <BYTES> - binary/latin1 string, length encoded in low 5 bits of tag */
|
68
|
+
#define SRL_HDR_SHORT_BINARY_LOW ((char)96)
|
69
|
+
#define SRL_HDR_SHORT_BINARY_HIGH ((char)127)
|
70
|
+
#define SRL_MASK_SHORT_BINARY_LEN ((char)31) /* mask to get length of SRL_HDR_SHORT_BINARY type tags */
|
71
|
+
|
72
|
+
#define SRL_HDR_TRACK_FLAG ((char)128) /* if this bit is set track the item */
|
73
|
+
|
data/ext/sereal/sereal.c
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
#include "sereal.h"
|
2
|
+
#include "encode.h"
|
3
|
+
|
4
|
+
VALUE Sereal = Qnil;
|
5
|
+
void Init_sereal();
|
6
|
+
void Init_sereal() {
|
7
|
+
Sereal = rb_define_class("Sereal", rb_cObject);
|
8
|
+
rb_define_singleton_method(Sereal, "encode", method_sereal_encode, -2);
|
9
|
+
rb_define_singleton_method(Sereal, "decode", method_sereal_decode, -2);
|
10
|
+
s_init_writers();
|
11
|
+
}
|
12
|
+
|
data/ext/sereal/sereal.h
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
#ifndef _MAIN_H
|
2
|
+
#define _MAIN_H
|
3
|
+
#include <ruby.h>
|
4
|
+
#include <ruby/encoding.h>
|
5
|
+
#include <ruby/st.h>
|
6
|
+
#include <string.h> /* memcpy,memset */
|
7
|
+
#include "proto.h"
|
8
|
+
|
9
|
+
typedef unsigned long long u64;
|
10
|
+
typedef unsigned int u32;
|
11
|
+
typedef unsigned short u16;
|
12
|
+
typedef unsigned char u8;
|
13
|
+
typedef struct _sereal sereal_t;
|
14
|
+
typedef struct _track_entry track_t;
|
15
|
+
|
16
|
+
#define TRUE 1
|
17
|
+
#define FALSE 0
|
18
|
+
#define MAX_RECURSION_DEPTH 100
|
19
|
+
#define COPY(src,dst,len) memcpy((dst),(src),len)
|
20
|
+
#define ZERO(src,len) memset((src),0,len)
|
21
|
+
|
22
|
+
#define is_ascii_string(str) (rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT)
|
23
|
+
#define THRESH(x,min,max) ((x) >= (min) && (x) <= (max))
|
24
|
+
|
25
|
+
#define IS_SHORT_BINARY(t) THRESH(t,SRL_HDR_SHORT_BINARY_LOW,SRL_HDR_SHORT_BINARY_HIGH)
|
26
|
+
#define IS_ARRAYREF(t) THRESH(t,SRL_HDR_ARRAYREF_LOW,SRL_HDR_ARRAYREF_HIGH)
|
27
|
+
#define IS_HASHREF(t) THRESH(t,SRL_HDR_HASHREF_LOW,SRL_HDR_HASHREF_HIGH)
|
28
|
+
#define IS_STRING(t) ((t) == SRL_HDR_STR_UTF8 || (t) == SRL_HDR_BINARY || IS_SHORT_BINARY((t)))
|
29
|
+
|
30
|
+
#define SRL_HDR_SYM SRL_HDR_RESERVED_LOW
|
31
|
+
#define SRL_HDR_RB_OBJ (SRL_HDR_RESERVED_LOW+1)
|
32
|
+
#define _D(fmt,arg...) fprintf(stderr,"%s(): " fmt "\n",__func__,##arg)
|
33
|
+
|
34
|
+
#ifdef ONIGURUMA_H
|
35
|
+
#define IGNORECASE ONIG_OPTION_IGNORECASE
|
36
|
+
#define MULTILINE ONIG_OPTION_MULTILINE
|
37
|
+
#define EXTENDED ONIG_OPTION_EXTEND
|
38
|
+
#else
|
39
|
+
#define IGNORECASE RE_OPTION_IGNORECASE
|
40
|
+
#define MULTILINE RE_OPTION_MULTILINE
|
41
|
+
#define EXTENDED RE_OPTION_EXTENDED
|
42
|
+
#endif
|
43
|
+
#define FLAG_SAFE 1
|
44
|
+
struct _sereal {
|
45
|
+
u8 *data;
|
46
|
+
u32 size;
|
47
|
+
u32 pos;
|
48
|
+
u32 rsize;
|
49
|
+
u32 level;
|
50
|
+
u8 flags;
|
51
|
+
};
|
52
|
+
|
53
|
+
VALUE method_sereal_encode(VALUE self, VALUE args);
|
54
|
+
VALUE method_sereal_decode(VALUE self, VALUE payload);
|
55
|
+
|
56
|
+
#define S_RECURSE_INC(s) \
|
57
|
+
do { \
|
58
|
+
if((s)->level++ > MAX_RECURSION_DEPTH) \
|
59
|
+
rb_raise(rb_eArgError, \
|
60
|
+
"max recursion depth reached: %d", \
|
61
|
+
MAX_RECURSION_DEPTH); \
|
62
|
+
} while(0);
|
63
|
+
|
64
|
+
#define S_RECURSE_DEC(s) ((s)->level--)
|
65
|
+
|
66
|
+
#ifndef HAVE_RB_INTERN_STR
|
67
|
+
#define rb_intern_str(string) SYM2ID(rb_str_intern(string))
|
68
|
+
#endif
|
69
|
+
#ifndef HAVE_RB_SYM_TO_S
|
70
|
+
#define rb_sym_to_s(object) rb_funcall(object,rb_intern("to_s"),0)
|
71
|
+
#endif
|
72
|
+
|
73
|
+
#endif
|
@@ -0,0 +1,129 @@
|
|
1
|
+
#ifndef __CSNAPPY_H__
|
2
|
+
#define __CSNAPPY_H__
|
3
|
+
/*
|
4
|
+
File modified for the Linux Kernel by
|
5
|
+
Zeev Tarantov <zeev.tarantov@gmail.com>
|
6
|
+
*/
|
7
|
+
#ifdef __cplusplus
|
8
|
+
extern "C" {
|
9
|
+
#endif
|
10
|
+
|
11
|
+
#define CSNAPPY_VERSION 4
|
12
|
+
|
13
|
+
#define CSNAPPY_WORKMEM_BYTES_POWER_OF_TWO 15
|
14
|
+
#define CSNAPPY_WORKMEM_BYTES (1 << CSNAPPY_WORKMEM_BYTES_POWER_OF_TWO)
|
15
|
+
|
16
|
+
#ifndef __GNUC__
|
17
|
+
#define __attribute__(x) /*NOTHING*/
|
18
|
+
#endif
|
19
|
+
|
20
|
+
/*
|
21
|
+
* Returns the maximal size of the compressed representation of
|
22
|
+
* input data that is "source_len" bytes in length;
|
23
|
+
*/
|
24
|
+
uint32_t
|
25
|
+
csnappy_max_compressed_length(uint32_t source_len) __attribute__((const));
|
26
|
+
|
27
|
+
/*
|
28
|
+
* Flat array compression that does not emit the "uncompressed length"
|
29
|
+
* prefix. Compresses "input" array to the "output" array.
|
30
|
+
*
|
31
|
+
* REQUIRES: "input" is at most 32KiB long.
|
32
|
+
* REQUIRES: "output" points to an array of memory that is at least
|
33
|
+
* "csnappy_max_compressed_length(input_length)" in size.
|
34
|
+
* REQUIRES: working_memory has (1 << workmem_bytes_power_of_two) bytes.
|
35
|
+
* REQUIRES: 9 <= workmem_bytes_power_of_two <= 15.
|
36
|
+
*
|
37
|
+
* Returns an "end" pointer into "output" buffer.
|
38
|
+
* "end - output" is the compressed size of "input".
|
39
|
+
*/
|
40
|
+
char*
|
41
|
+
csnappy_compress_fragment(
|
42
|
+
const char *input,
|
43
|
+
const uint32_t input_length,
|
44
|
+
char *output,
|
45
|
+
void *working_memory,
|
46
|
+
const int workmem_bytes_power_of_two);
|
47
|
+
|
48
|
+
/*
|
49
|
+
* REQUIRES: "compressed" must point to an area of memory that is at
|
50
|
+
* least "csnappy_max_compressed_length(input_length)" bytes in length.
|
51
|
+
* REQUIRES: working_memory has (1 << workmem_bytes_power_of_two) bytes.
|
52
|
+
* REQUIRES: 9 <= workmem_bytes_power_of_two <= 15.
|
53
|
+
*
|
54
|
+
* Takes the data stored in "input[0..input_length-1]" and stores
|
55
|
+
* it in the array pointed to by "compressed".
|
56
|
+
*
|
57
|
+
* "*out_compressed_length" is set to the length of the compressed output.
|
58
|
+
*/
|
59
|
+
void
|
60
|
+
csnappy_compress(
|
61
|
+
const char *input,
|
62
|
+
uint32_t input_length,
|
63
|
+
char *compressed,
|
64
|
+
uint32_t *out_compressed_length,
|
65
|
+
void *working_memory,
|
66
|
+
const int workmem_bytes_power_of_two);
|
67
|
+
|
68
|
+
/*
|
69
|
+
* Reads header of compressed data to get stored length of uncompressed data.
|
70
|
+
* REQUIRES: start points to compressed data.
|
71
|
+
* REQUIRES: n is length of available compressed data.
|
72
|
+
*
|
73
|
+
* Returns SNAPPY_E_HEADER_BAD on error.
|
74
|
+
* Returns number of bytes read from input on success.
|
75
|
+
* Stores decoded length into *result.
|
76
|
+
*/
|
77
|
+
int
|
78
|
+
csnappy_get_uncompressed_length(
|
79
|
+
const char *start,
|
80
|
+
uint32_t n,
|
81
|
+
uint32_t *result);
|
82
|
+
|
83
|
+
/*
|
84
|
+
* Safely decompresses all data from array "src" of length "src_len" containing
|
85
|
+
* entire compressed stream (with header) into array "dst" of size "dst_len".
|
86
|
+
* REQUIRES: dst_len is at least csnappy_get_uncompressed_length(...).
|
87
|
+
*
|
88
|
+
* Iff successful, returns CSNAPPY_E_OK.
|
89
|
+
* If recorded length in header is greater than dst_len, returns
|
90
|
+
* CSNAPPY_E_OUTPUT_INSUF.
|
91
|
+
* If compressed data is malformed, does not write more than dst_len into dst.
|
92
|
+
*/
|
93
|
+
int
|
94
|
+
csnappy_decompress(
|
95
|
+
const char *src,
|
96
|
+
uint32_t src_len,
|
97
|
+
char *dst,
|
98
|
+
uint32_t dst_len);
|
99
|
+
|
100
|
+
/*
|
101
|
+
* Safely decompresses stream src_len bytes long read from src to dst.
|
102
|
+
* Amount of available space at dst must be provided in *dst_len by caller.
|
103
|
+
* If compressed stream needs more space, it will not overflow and return
|
104
|
+
* CSNAPPY_E_OUTPUT_OVERRUN.
|
105
|
+
* On success, sets *dst_len to actal number of bytes decompressed.
|
106
|
+
* Iff successful, returns CSNAPPY_E_OK.
|
107
|
+
*/
|
108
|
+
int
|
109
|
+
csnappy_decompress_noheader(
|
110
|
+
const char *src,
|
111
|
+
uint32_t src_len,
|
112
|
+
char *dst,
|
113
|
+
uint32_t *dst_len);
|
114
|
+
|
115
|
+
/*
|
116
|
+
* Return values (< 0 = Error)
|
117
|
+
*/
|
118
|
+
#define CSNAPPY_E_OK 0
|
119
|
+
#define CSNAPPY_E_HEADER_BAD (-1)
|
120
|
+
#define CSNAPPY_E_OUTPUT_INSUF (-2)
|
121
|
+
#define CSNAPPY_E_OUTPUT_OVERRUN (-3)
|
122
|
+
#define CSNAPPY_E_INPUT_NOT_CONSUMED (-4)
|
123
|
+
#define CSNAPPY_E_DATA_MALFORMED (-5)
|
124
|
+
|
125
|
+
#ifdef __cplusplus
|
126
|
+
}
|
127
|
+
#endif
|
128
|
+
|
129
|
+
#endif
|