hirlite 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +15 -0
  2. data/LICENSE +28 -0
  3. data/Rakefile +51 -0
  4. data/ext/hirlite_ext/extconf.rb +33 -0
  5. data/ext/hirlite_ext/hirlite_ext.c +14 -0
  6. data/ext/hirlite_ext/hirlite_ext.h +38 -0
  7. data/ext/hirlite_ext/rlite.c +351 -0
  8. data/lib/hirlite/rlite.rb +1 -0
  9. data/lib/hirlite/version.rb +3 -0
  10. data/lib/hirlite.rb +2 -0
  11. data/vendor/rlite/Makefile +6 -0
  12. data/vendor/rlite/deps/crc64.c +191 -0
  13. data/vendor/rlite/deps/crc64.h +3 -0
  14. data/vendor/rlite/deps/endianconv.h +73 -0
  15. data/vendor/rlite/deps/hyperloglog.c +1547 -0
  16. data/vendor/rlite/deps/hyperloglog.h +14 -0
  17. data/vendor/rlite/deps/lzf.h +100 -0
  18. data/vendor/rlite/deps/lzfP.h +159 -0
  19. data/vendor/rlite/deps/lzf_c.c +295 -0
  20. data/vendor/rlite/deps/lzf_d.c +150 -0
  21. data/vendor/rlite/deps/sha1.c +227 -0
  22. data/vendor/rlite/deps/sha1.h +19 -0
  23. data/vendor/rlite/deps/utilfromredis.c +397 -0
  24. data/vendor/rlite/deps/utilfromredis.h +11 -0
  25. data/vendor/rlite/src/Makefile +79 -0
  26. data/vendor/rlite/src/constants.h +15 -0
  27. data/vendor/rlite/src/dump.c +191 -0
  28. data/vendor/rlite/src/dump.h +3 -0
  29. data/vendor/rlite/src/hirlite.c +3985 -0
  30. data/vendor/rlite/src/hirlite.h +186 -0
  31. data/vendor/rlite/src/page_btree.c +1556 -0
  32. data/vendor/rlite/src/page_btree.h +133 -0
  33. data/vendor/rlite/src/page_key.c +283 -0
  34. data/vendor/rlite/src/page_key.h +25 -0
  35. data/vendor/rlite/src/page_list.c +718 -0
  36. data/vendor/rlite/src/page_list.h +70 -0
  37. data/vendor/rlite/src/page_long.c +61 -0
  38. data/vendor/rlite/src/page_long.h +14 -0
  39. data/vendor/rlite/src/page_multi_string.c +538 -0
  40. data/vendor/rlite/src/page_multi_string.h +18 -0
  41. data/vendor/rlite/src/page_skiplist.c +689 -0
  42. data/vendor/rlite/src/page_skiplist.h +70 -0
  43. data/vendor/rlite/src/page_string.c +55 -0
  44. data/vendor/rlite/src/page_string.h +12 -0
  45. data/vendor/rlite/src/pqsort.c +185 -0
  46. data/vendor/rlite/src/pqsort.h +40 -0
  47. data/vendor/rlite/src/restore.c +401 -0
  48. data/vendor/rlite/src/restore.h +3 -0
  49. data/vendor/rlite/src/rlite.c +1309 -0
  50. data/vendor/rlite/src/rlite.h +159 -0
  51. data/vendor/rlite/src/sort.c +530 -0
  52. data/vendor/rlite/src/sort.h +18 -0
  53. data/vendor/rlite/src/status.h +19 -0
  54. data/vendor/rlite/src/type_hash.c +607 -0
  55. data/vendor/rlite/src/type_hash.h +29 -0
  56. data/vendor/rlite/src/type_list.c +477 -0
  57. data/vendor/rlite/src/type_list.h +23 -0
  58. data/vendor/rlite/src/type_set.c +796 -0
  59. data/vendor/rlite/src/type_set.h +34 -0
  60. data/vendor/rlite/src/type_string.c +613 -0
  61. data/vendor/rlite/src/type_string.h +34 -0
  62. data/vendor/rlite/src/type_zset.c +1147 -0
  63. data/vendor/rlite/src/type_zset.h +50 -0
  64. data/vendor/rlite/src/util.c +334 -0
  65. data/vendor/rlite/src/util.h +71 -0
  66. metadata +151 -0
@@ -0,0 +1,401 @@
1
+ #include <arpa/inet.h>
2
+ #include "page_key.h"
3
+ #include "rlite.h"
4
+ #include "util.h"
5
+ #include "../deps/crc64.h"
6
+ #include "../deps/endianconv.h"
7
+ #include "../deps/lzf.h"
8
+
9
+ static int verify(unsigned char *data, long datalen) {
10
+ unsigned char *footer;
11
+ uint16_t rdbver;
12
+ uint64_t crc;
13
+
14
+ /* At least 2 bytes of RDB version and 8 of CRC64 should be present. */
15
+ if (datalen < 10) return RL_INVALID_PARAMETERS;
16
+ footer = data+(datalen-10);
17
+
18
+ /* Verify RDB version */
19
+ rdbver = (footer[1] << 8) | footer[0];
20
+ if (rdbver != REDIS_RDB_VERSION) return RL_INVALID_PARAMETERS;
21
+
22
+ /* Verify CRC64 */
23
+ crc = rl_crc64(0,data,datalen-8);
24
+ memrev64ifbe(&crc);
25
+ return (memcmp(&crc,footer+2,8) == 0) ? RL_OK : RL_INVALID_PARAMETERS;
26
+ }
27
+
28
+ static unsigned char *read_signed_short(unsigned char *data, long *value) {
29
+ *value = (char)data[0] | (char)(data[1] << 8);
30
+ return data + 2;
31
+ }
32
+
33
+ static unsigned char *read_signed_int(unsigned char *data, long *value) {
34
+ *value = (long)data[0] | (long)(data[1] << 8) | (long)(data[2] << 16) | (long)(data[3] << 24);
35
+ return data + 4;
36
+ }
37
+
38
+ static unsigned char *read_signed_long(unsigned char *data, long *value) {
39
+ long tmp;
40
+ *value = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24);
41
+ tmp = data[4];
42
+ *value |= tmp << 32;
43
+ tmp = data[5];
44
+ *value |= tmp << 40;
45
+ tmp = data[6];
46
+ *value |= tmp << 48;
47
+ tmp = data[7];
48
+ *value |= tmp << 56;
49
+ return data + 8;
50
+ }
51
+
52
+ static unsigned char *read_unsigned_short(unsigned char *data, unsigned long *value) {
53
+ *value = data[0] | (data[1] << 8);
54
+ return data + 2;
55
+ }
56
+
57
+ static unsigned char *read_unsigned_int(unsigned char *data, unsigned long *value) {
58
+ *value = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24);
59
+ return data + 4;
60
+ }
61
+
62
+ static unsigned char *read_unsigned_long(unsigned char *data, unsigned long *value) {
63
+ unsigned long tmp;
64
+ *value = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24);
65
+ tmp = data[4];
66
+ *value |= tmp << 32;
67
+ tmp = data[5];
68
+ *value |= tmp << 40;
69
+ tmp = data[6];
70
+ *value |= tmp << 48;
71
+ tmp = data[7];
72
+ *value |= tmp << 56;
73
+ return data + 8;
74
+ }
75
+
76
+ static unsigned char *read_length_with_encoding(unsigned char *f, long *length, int *is_encoded)
77
+ {
78
+ int enc_type = (f[0] & 0xC0) >> 6;
79
+ if (enc_type == REDIS_RDB_ENCVAL) {
80
+ if (is_encoded) {
81
+ *is_encoded = 1;
82
+ }
83
+ *length = f[0] & 0x3F;
84
+ return f + 1;
85
+ } else if (enc_type == REDIS_RDB_6BITLEN) {
86
+ if (is_encoded) {
87
+ *is_encoded = 0;
88
+ }
89
+ *length = f[0] & 0x3F;
90
+ return f + 1;
91
+ } else if (enc_type == REDIS_RDB_14BITLEN) {
92
+ if (is_encoded) {
93
+ *is_encoded = 0;
94
+ }
95
+ *length = ((f[0] & 0x3F) << 8 ) | f[1];
96
+ return f + 2;
97
+ } else {
98
+ if (is_encoded) {
99
+ *is_encoded = 0;
100
+ }
101
+ *length = ntohl(*(uint32_t*)(f + 1));
102
+ return f + 5;
103
+ }
104
+ }
105
+
106
+ static unsigned char *read_ziplist_entry(unsigned char *data, unsigned char **_entry, long *_length)
107
+ {
108
+ long length = 0;
109
+ unsigned char *entry;
110
+ unsigned long prev_length = data[0];
111
+ data++;
112
+ if (prev_length == 254) {
113
+ data = read_unsigned_int(data, &prev_length);
114
+ }
115
+ unsigned long entry_header = data[0];
116
+ if ((entry_header >> 6) == 0) {
117
+ length = entry_header & 0x3F;
118
+ data++;
119
+ }
120
+ else if ((entry_header >> 6) == 1) {
121
+ length = ((entry_header & 0x3F) << 8) | data[0];
122
+ data++;
123
+ }
124
+ else if ((entry_header >> 6) == 2) {
125
+ // TODO: length = read_big_endian_unsigned_int(f)
126
+ data = NULL;
127
+ } else if ((entry_header >> 4) == 12) {
128
+ data = read_signed_short(data, &length);
129
+ } else if ((entry_header >> 4) == 13) {
130
+ data = read_signed_int(data, &length);
131
+ } else if ((entry_header >> 4) == 14) {
132
+ data = read_signed_long(data, &length);
133
+ } else if (entry_header == 240) {
134
+ unsigned char tmp[5];
135
+ tmp[0] = tmp[4] = 0;
136
+ memcpy(&tmp[1], data, 3);
137
+ data = read_signed_int(data, &length);
138
+ } else if (entry_header == 254) {
139
+ length = data[0];
140
+ data++;
141
+ } else if (entry_header >= 241 && entry_header <= 253) {
142
+ entry = rl_malloc(sizeof(unsigned char) * 2);
143
+ if (!entry) {
144
+ return NULL;
145
+ }
146
+ length = snprintf((char *)entry, 2, "%lu", entry_header - 241);
147
+ data++;
148
+ goto ret;
149
+ }
150
+ entry = rl_malloc(sizeof(unsigned char) * length);
151
+ if (!entry) {
152
+ return NULL;
153
+ }
154
+ memcpy(entry, data, length);
155
+ data += length;
156
+ ret:
157
+ *_entry = entry;
158
+ *_length = length;
159
+ return data;
160
+ }
161
+
162
+ static int read_string(unsigned char *data, unsigned char **str, long *strlen, unsigned char **newdata)
163
+ {
164
+ int retval = RL_OK;
165
+ long length, strdatalen, cdatalen;
166
+ unsigned char *strdata;
167
+ int is_encoded;
168
+ data = read_length_with_encoding(data, &length, &is_encoded);
169
+ if (is_encoded && length == REDIS_RDB_ENC_INT8) {
170
+ strdatalen = 5;
171
+ RL_MALLOC(strdata, strdatalen * sizeof(unsigned char));
172
+ strdatalen = snprintf((char *)strdata, strdatalen, "%d", (signed char)data[0]);
173
+ if (strdatalen < 0) {
174
+ retval = RL_UNEXPECTED;
175
+ goto cleanup;
176
+ }
177
+ data++;
178
+ } else if (is_encoded && length == REDIS_RDB_ENC_INT16) {
179
+ strdatalen = 7;
180
+ strdata = rl_malloc(strdatalen * sizeof(unsigned char));
181
+ strdatalen = snprintf((char *)strdata, strdatalen, "%d", (int16_t)(data[0] | (data[1] << 8)));
182
+ if (strdatalen < 0) {
183
+ retval = RL_UNEXPECTED;
184
+ goto cleanup;
185
+ }
186
+ data += 2;
187
+ } else if (is_encoded && length == REDIS_RDB_ENC_INT32) {
188
+ strdatalen = 12;
189
+ strdata = rl_malloc(strdatalen * sizeof(unsigned char));
190
+ strdatalen = snprintf((char *)strdata, strdatalen, "%d", (int32_t)(data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24)));
191
+ if (strdatalen < 0) {
192
+ retval = RL_UNEXPECTED;
193
+ goto cleanup;
194
+ }
195
+ data += 4;
196
+ } else if (is_encoded && length == REDIS_RDB_ENC_LZF) {
197
+ data = read_length_with_encoding(data, &cdatalen, NULL);
198
+ data = read_length_with_encoding(data, &strdatalen, NULL);
199
+ strdata = rl_malloc(strdatalen * sizeof(unsigned char));
200
+ rl_lzf_decompress(data, cdatalen, strdata, strdatalen);
201
+ data += cdatalen;
202
+ } else if (!is_encoded) {
203
+ strdatalen = length;
204
+ strdata = rl_malloc(strdatalen * sizeof(unsigned char));
205
+ memcpy(strdata, data, strdatalen);
206
+ data += strdatalen;
207
+ } else {
208
+ retval = RL_NOT_IMPLEMENTED;
209
+ goto cleanup;
210
+ }
211
+ *str = strdata;
212
+ *strlen = strdatalen;
213
+ if (newdata) {
214
+ *newdata = data;
215
+ }
216
+ cleanup:
217
+ return retval;
218
+ }
219
+
220
+ int rl_restore(struct rlite *db, const unsigned char *key, long keylen, unsigned long long expires, unsigned char *_data, long datalen)
221
+ {
222
+ int retval;
223
+ unsigned char type;
224
+ long i, length, length2;
225
+ unsigned char *strdata = NULL, *strdata2 = NULL, *strdata3 = NULL;
226
+ unsigned char *data = _data, *tmpdata;
227
+ long strdatalen = 0, strdata2len, strdata3len;
228
+ unsigned long j, encoding, numentries, ulvalue;
229
+ void **tmp = NULL;
230
+ char f[40];
231
+ double d;
232
+
233
+ RL_CALL(rl_key_get, RL_NOT_FOUND, db, key, keylen, NULL, NULL, NULL, NULL, NULL);
234
+ RL_CALL(verify, RL_OK, data, datalen);
235
+
236
+ type = data[0];
237
+ data++;
238
+
239
+ if (type == REDIS_RDB_TYPE_STRING) {
240
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, NULL);
241
+ RL_CALL(rl_set, RL_OK, db, key, keylen, strdata, strdatalen, 1, expires);
242
+ }
243
+ else if (type == REDIS_RDB_TYPE_LIST) {
244
+ data = read_length_with_encoding(data, &length, NULL);
245
+ for (i = 0; i < length; i++) {
246
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
247
+ RL_CALL(rl_push, RL_OK, db, key, keylen, 1, 0, 1, &strdata, &strdatalen, NULL);
248
+ rl_free(strdata);
249
+ strdata = NULL;
250
+ }
251
+ }
252
+ else if (type == REDIS_RDB_TYPE_SET) {
253
+ data = read_length_with_encoding(data, &length, NULL);
254
+ for (i = 0; i < length; i++) {
255
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
256
+ RL_CALL(rl_sadd, RL_OK, db, key, keylen, 1, &strdata, &strdatalen, NULL);
257
+ rl_free(strdata);
258
+ strdata = NULL;
259
+ }
260
+ }
261
+ else if (type == REDIS_RDB_TYPE_ZSET) {
262
+ data = read_length_with_encoding(data, &length, NULL);
263
+ for (i = 0; i < length; i++) {
264
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
265
+ length2 = data[0];
266
+ data++;
267
+ if (length2 > 40 || length2 < 1) {
268
+ retval = RL_UNEXPECTED;
269
+ goto cleanup;
270
+ }
271
+ memcpy(f, data, length2);
272
+ data += length2;
273
+ f[length2] = 0;
274
+ d = strtold(f, NULL);
275
+
276
+ RL_CALL(rl_zadd, RL_OK, db, key, keylen, d, strdata, strdatalen);
277
+ rl_free(strdata);
278
+ strdata = NULL;
279
+ }
280
+ }
281
+ else if (type == REDIS_RDB_TYPE_HASH) {
282
+ data = read_length_with_encoding(data, &length, NULL);
283
+ for (i = 0; i < length; i++) {
284
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
285
+ RL_CALL(read_string, RL_OK, data, &strdata2, &length2, &data);
286
+
287
+ RL_CALL(rl_hset, RL_OK, db, key, keylen, strdata, strdatalen, strdata2, length2, NULL, 0);
288
+ rl_free(strdata);
289
+ strdata = NULL;
290
+ rl_free(strdata2);
291
+ strdata2 = NULL;
292
+ }
293
+ }
294
+ else if (type == REDIS_RDB_TYPE_HASH_ZIPMAP) {
295
+ retval = RL_NOT_IMPLEMENTED;
296
+ goto cleanup;
297
+ }
298
+ else if (type == REDIS_RDB_TYPE_LIST_ZIPLIST) {
299
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
300
+ tmpdata = strdata + 10;
301
+ while (*tmpdata != 255) {
302
+ tmpdata = read_ziplist_entry(tmpdata, &strdata2, &strdata2len);
303
+ if (!tmpdata) {
304
+ retval = RL_UNEXPECTED;
305
+ goto cleanup;
306
+ }
307
+ RL_CALL(rl_push, RL_OK, db, key, keylen, 1, 0, 1, &strdata2, &strdata2len, NULL);
308
+ rl_free(strdata2);
309
+ strdata2 = NULL;
310
+ }
311
+ rl_free(strdata);
312
+ strdata = NULL;
313
+ }
314
+ else if (type == REDIS_RDB_TYPE_SET_INTSET) {
315
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
316
+ tmpdata = strdata;
317
+ tmpdata = read_unsigned_int(tmpdata, &encoding);
318
+ tmpdata = read_unsigned_int(tmpdata, &numentries);
319
+ if (encoding != 2 && encoding != 4 && encoding != 8) {
320
+ retval = RL_UNEXPECTED;
321
+ goto cleanup;
322
+ }
323
+ RL_MALLOC(tmp, sizeof(void *));
324
+ for (j = 0; j < numentries; j++) {
325
+ if (encoding == 8) {
326
+ tmpdata = read_unsigned_long(tmpdata, &ulvalue);
327
+ } else if (encoding == 4) {
328
+ tmpdata = read_unsigned_int(tmpdata, &ulvalue);
329
+ } else if (encoding == 2) {
330
+ tmpdata = read_unsigned_short(tmpdata, &ulvalue);
331
+ }
332
+ length2 = snprintf(f, 40, "%lu", ulvalue);
333
+ tmp[0] = f;
334
+ RL_CALL(rl_sadd, RL_OK, db, key, keylen, 1, (unsigned char **)tmp, &length2, NULL);
335
+ }
336
+ }
337
+ else if (type == REDIS_RDB_TYPE_ZSET_ZIPLIST) {
338
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
339
+ tmpdata = strdata + 10;
340
+ while (*tmpdata != 255) {
341
+ tmpdata = read_ziplist_entry(tmpdata, &strdata2, &strdata2len);
342
+ if (!tmpdata) {
343
+ retval = RL_UNEXPECTED;
344
+ goto cleanup;
345
+ }
346
+ tmpdata = read_ziplist_entry(tmpdata, &strdata3, &strdata3len);
347
+ if (!tmpdata) {
348
+ retval = RL_UNEXPECTED;
349
+ goto cleanup;
350
+ }
351
+
352
+ if (strdata3len > 40 || strdata3len < 1) {
353
+ retval = RL_UNEXPECTED;
354
+ goto cleanup;
355
+ }
356
+ memcpy(f, strdata3, strdata3len);
357
+ f[strdata3len] = 0;
358
+ d = strtold(f, NULL);
359
+ RL_CALL(rl_zadd, RL_OK, db, key, keylen, d, strdata2, strdata2len);
360
+ rl_free(strdata2);
361
+ strdata2 = NULL;
362
+ rl_free(strdata3);
363
+ strdata3 = NULL;
364
+ }
365
+ rl_free(strdata);
366
+ strdata = NULL;
367
+ }
368
+ else if (type == REDIS_RDB_TYPE_HASH_ZIPLIST) {
369
+ RL_CALL(read_string, RL_OK, data, &strdata, &strdatalen, &data);
370
+ tmpdata = strdata + 10;
371
+ while (*tmpdata != 255) {
372
+ tmpdata = read_ziplist_entry(tmpdata, &strdata2, &strdata2len);
373
+ if (!tmpdata) {
374
+ retval = RL_UNEXPECTED;
375
+ goto cleanup;
376
+ }
377
+ tmpdata = read_ziplist_entry(tmpdata, &strdata3, &strdata3len);
378
+ if (!tmpdata) {
379
+ retval = RL_UNEXPECTED;
380
+ goto cleanup;
381
+ }
382
+ RL_CALL(rl_hset, RL_OK, db, key, keylen, strdata2, strdata2len, strdata3, strdata3len, NULL, 0);
383
+ rl_free(strdata2);
384
+ strdata2 = NULL;
385
+ rl_free(strdata3);
386
+ strdata3 = NULL;
387
+ }
388
+ rl_free(strdata);
389
+ strdata = NULL;
390
+ } else {
391
+ retval = RL_NOT_IMPLEMENTED;
392
+ goto cleanup;
393
+ }
394
+ retval = RL_OK;
395
+ cleanup:
396
+ rl_free(tmp);
397
+ rl_free(strdata);
398
+ rl_free(strdata2);
399
+ rl_free(strdata3);
400
+ return retval;
401
+ }
@@ -0,0 +1,3 @@
1
+ #include "rlite.h"
2
+
3
+ int rl_restore(struct rlite *db, const unsigned char *key, long keylen, unsigned long long expires, unsigned char *data, long datalen);