sereal 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/sereal/buffer.h +15 -8
- data/ext/sereal/decode.c +72 -28
- data/ext/sereal/encode.c +107 -18
- data/ext/sereal/lz4.c +822 -0
- data/ext/sereal/lz4.h +205 -0
- data/ext/sereal/lz4/lz4.c +822 -0
- data/ext/sereal/lz4/lz4.h +205 -0
- data/ext/sereal/lz4/lz4hc.c +817 -0
- data/ext/sereal/lz4/lz4hc.h +111 -0
- data/ext/sereal/lz4hc.c +817 -0
- data/ext/sereal/lz4hc.h +111 -0
- data/ext/sereal/proto.h +4 -2
- data/ext/sereal/sereal.c +30 -0
- data/ext/sereal/sereal.h +13 -1
- data/ext/sereal/snappy/csnappy_compat.h +16 -0
- data/ext/sereal/snappy/csnappy_compress.c +19 -16
- data/ext/sereal/snappy/csnappy_decompress.c +8 -5
- data/ext/sereal/snappy/csnappy_internal.h +12 -3
- data/ext/sereal/snappy/csnappy_internal_userspace.h +56 -18
- metadata +11 -2
data/ext/sereal/lz4hc.h
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
/*
|
2
|
+
LZ4 HC - High Compression Mode of LZ4
|
3
|
+
Header File
|
4
|
+
Copyright (C) 2011-2013, Yann Collet.
|
5
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
6
|
+
|
7
|
+
Redistribution and use in source and binary forms, with or without
|
8
|
+
modification, are permitted provided that the following conditions are
|
9
|
+
met:
|
10
|
+
|
11
|
+
* Redistributions of source code must retain the above copyright
|
12
|
+
notice, this list of conditions and the following disclaimer.
|
13
|
+
* Redistributions in binary form must reproduce the above
|
14
|
+
copyright notice, this list of conditions and the following disclaimer
|
15
|
+
in the documentation and/or other materials provided with the
|
16
|
+
distribution.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
+
|
30
|
+
You can contact the author at :
|
31
|
+
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
32
|
+
- LZ4 source repository : http://code.google.com/p/lz4/
|
33
|
+
*/
|
34
|
+
#pragma once
|
35
|
+
|
36
|
+
|
37
|
+
#if defined (__cplusplus)
|
38
|
+
extern "C" {
|
39
|
+
#endif
|
40
|
+
|
41
|
+
|
42
|
+
int LZ4_compressHC (const char* source, char* dest, int inputSize);
|
43
|
+
/*
|
44
|
+
LZ4_compressHC :
|
45
|
+
return : the number of bytes in compressed buffer dest
|
46
|
+
or 0 if compression fails.
|
47
|
+
note : destination buffer must be already allocated.
|
48
|
+
To avoid any problem, size it to handle worst cases situations (input data not compressible)
|
49
|
+
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
|
50
|
+
*/
|
51
|
+
|
52
|
+
int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
|
53
|
+
/*
|
54
|
+
LZ4_compress_limitedOutput() :
|
55
|
+
Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
|
56
|
+
If it cannot achieve it, compression will stop, and result of the function will be zero.
|
57
|
+
This function never writes outside of provided output buffer.
|
58
|
+
|
59
|
+
inputSize : Max supported value is 1 GB
|
60
|
+
maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
|
61
|
+
return : the number of output bytes written in buffer 'dest'
|
62
|
+
or 0 if compression fails.
|
63
|
+
*/
|
64
|
+
|
65
|
+
|
66
|
+
/* Note :
|
67
|
+
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
|
68
|
+
*/
|
69
|
+
|
70
|
+
|
71
|
+
/* Advanced Functions */
|
72
|
+
|
73
|
+
void* LZ4_createHC (const char* inputBuffer);
|
74
|
+
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
|
75
|
+
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
|
76
|
+
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
|
77
|
+
int LZ4_freeHC (void* LZ4HC_Data);
|
78
|
+
|
79
|
+
/*
|
80
|
+
These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
|
81
|
+
In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
|
82
|
+
|
83
|
+
void* LZ4_createHC (const char* inputBuffer);
|
84
|
+
The result of the function is the (void*) pointer on the LZ4HC Data Structure.
|
85
|
+
This pointer will be needed in all other functions.
|
86
|
+
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
|
87
|
+
The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
|
88
|
+
The input buffer must be already allocated, and size at least 192KB.
|
89
|
+
'inputBuffer' will also be the 'const char* source' of the first block.
|
90
|
+
|
91
|
+
All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
|
92
|
+
To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
|
93
|
+
Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
|
94
|
+
but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
|
95
|
+
If next block does not begin immediately after the previous one, the compression will fail (return 0).
|
96
|
+
|
97
|
+
When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
|
98
|
+
char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
|
99
|
+
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
|
100
|
+
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
|
101
|
+
==> The memory position where the next input data block must start is provided as the result of the function.
|
102
|
+
|
103
|
+
Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
|
104
|
+
|
105
|
+
When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
|
106
|
+
*/
|
107
|
+
|
108
|
+
|
109
|
+
#if defined (__cplusplus)
|
110
|
+
}
|
111
|
+
#endif
|
data/ext/sereal/proto.h
CHANGED
@@ -6,8 +6,10 @@
|
|
6
6
|
#define SRL_PROTOCOL_VERSION_MASK ( ( 1 << SRL_PROTOCOL_VERSION_BITS ) - 1 )
|
7
7
|
|
8
8
|
#define SRL_PROTOCOL_ENCODING_MASK ( ~SRL_PROTOCOL_VERSION_MASK )
|
9
|
-
#define SRL_PROTOCOL_ENCODING_RAW
|
10
|
-
#define SRL_PROTOCOL_ENCODING_SNAPPY
|
9
|
+
#define SRL_PROTOCOL_ENCODING_RAW ( 0 << SRL_PROTOCOL_VERSION_BITS )
|
10
|
+
#define SRL_PROTOCOL_ENCODING_SNAPPY ( 1 << SRL_PROTOCOL_VERSION_BITS )
|
11
|
+
#define SRL_PROTOCOL_ENCODING_SNAPPY_INCR ( 2 << SRL_PROTOCOL_VERSION_BITS )
|
12
|
+
#define SRL_PROTOCOL_ENCODING_LZ4_INCR ( 3 << SRL_PROTOCOL_VERSION_BITS )
|
11
13
|
|
12
14
|
#define SRL_HDR_POS ((char)0) /* small positive integer - value in low 4 bits (identity) */
|
13
15
|
#define SRL_HDR_POS_LOW ((char)0) /* small positive integer - value in low 4 bits (identity) */
|
data/ext/sereal/sereal.c
CHANGED
@@ -3,10 +3,40 @@
|
|
3
3
|
|
4
4
|
VALUE Sereal = Qnil;
|
5
5
|
void Init_sereal();
|
6
|
+
|
7
|
+
/*
|
8
|
+
* Encode/Decode object using Sereal binary protocol:
|
9
|
+
* https://github.com/Sereal/Sereal/blob/master/sereal_spec.pod
|
10
|
+
*
|
11
|
+
* Sereal.encode(object) -> serialized blob
|
12
|
+
* Sereal.encode(object,Sereal::LZ4) -> LZ4 compressed blob
|
13
|
+
* Sereal.encode(object,Sereal::LZ4HC) -> LZ4HC compressed blob
|
14
|
+
* Sereal.encode(object,Sereal::SNAPPY_INCR) -> snappy compressed blob
|
15
|
+
* Sereal.encode(object,Sereal::SNAPPY) -> snappy compressed blob
|
16
|
+
*
|
17
|
+
* LZ4 LZ4HC and SNAPPY_INCR can be appended into one output and then the
|
18
|
+
* decoder will know what to do.
|
19
|
+
*
|
20
|
+
* Sereal.decode(blob) - returns the decoded object
|
21
|
+
*
|
22
|
+
* If the blob contains multiple compressed(with LZ4* or SNAPPY_INCR)
|
23
|
+
* sub-blobs you should call it with:
|
24
|
+
*
|
25
|
+
* Sereal.decode(blob) do |decoded|
|
26
|
+
* # do something with the decoded object
|
27
|
+
* end
|
28
|
+
* otherwise only the first decoded object will be returned
|
29
|
+
*
|
30
|
+
*/
|
6
31
|
void Init_sereal() {
|
7
32
|
Sereal = rb_define_class("Sereal", rb_cObject);
|
8
33
|
rb_define_singleton_method(Sereal, "encode", method_sereal_encode, -2);
|
9
34
|
rb_define_singleton_method(Sereal, "decode", method_sereal_decode, -2);
|
35
|
+
rb_define_const(Sereal, "SNAPPY",INT2NUM(__SNAPPY));
|
36
|
+
rb_define_const(Sereal, "SNAPPY_INCR",INT2NUM(__SNAPPY_INCR));
|
37
|
+
rb_define_const(Sereal, "RAW",INT2NUM(__RAW));
|
38
|
+
rb_define_const(Sereal, "LZ4",INT2NUM(__LZ4_INCR));
|
39
|
+
rb_define_const(Sereal, "LZ4HC",INT2NUM(__LZ4HC_INCR));
|
10
40
|
s_init_writers();
|
11
41
|
}
|
12
42
|
|
data/ext/sereal/sereal.h
CHANGED
@@ -40,7 +40,13 @@ typedef struct _track_entry track_t;
|
|
40
40
|
#define MULTILINE RE_OPTION_MULTILINE
|
41
41
|
#define EXTENDED RE_OPTION_EXTENDED
|
42
42
|
#endif
|
43
|
-
#define
|
43
|
+
#define s_raise(what,ex,arg...) \
|
44
|
+
do { \
|
45
|
+
s_destroy(what); \
|
46
|
+
rb_raise(ex,##arg); \
|
47
|
+
} while(0);
|
48
|
+
|
49
|
+
#define FLAG_NOT_MINE 1
|
44
50
|
struct _sereal {
|
45
51
|
u8 *data;
|
46
52
|
u32 size;
|
@@ -70,4 +76,10 @@ do { \
|
|
70
76
|
#define rb_sym_to_s(object) rb_funcall(object,rb_intern("to_s"),0)
|
71
77
|
#endif
|
72
78
|
|
79
|
+
#define __RAW 0
|
80
|
+
#define __SNAPPY 1
|
81
|
+
#define __SNAPPY_INCR 2
|
82
|
+
#define __LZ4_INCR 3
|
83
|
+
#define __LZ4HC_INCR 4
|
84
|
+
|
73
85
|
#endif
|
@@ -0,0 +1,16 @@
|
|
1
|
+
#ifndef CSNAPPY_COMPAT_H
|
2
|
+
|
3
|
+
/* This file was added to Sereal to attempt some MSVC compatibility,
|
4
|
+
* but is at best a band-aid. And done without a lot of experience
|
5
|
+
* in whatever subset of C99 MSVC supports.
|
6
|
+
*/
|
7
|
+
|
8
|
+
#ifndef INLINE
|
9
|
+
# if defined(_MSC_VER)
|
10
|
+
# define INLINE __inline
|
11
|
+
# else
|
12
|
+
# define INLINE inline
|
13
|
+
# endif
|
14
|
+
#endif
|
15
|
+
|
16
|
+
#endif
|
@@ -30,6 +30,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30
30
|
|
31
31
|
File modified for the Linux Kernel by
|
32
32
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
33
|
+
|
34
|
+
File modified for Sereal by
|
35
|
+
Steffen Mueller <smueller@cpan.org>
|
33
36
|
*/
|
34
37
|
|
35
38
|
#include "csnappy_internal.h"
|
@@ -40,7 +43,7 @@ Zeev Tarantov <zeev.tarantov@gmail.com>
|
|
40
43
|
#include "csnappy.h"
|
41
44
|
|
42
45
|
|
43
|
-
static
|
46
|
+
static INLINE char*
|
44
47
|
encode_varint32(char *sptr, uint32_t v)
|
45
48
|
{
|
46
49
|
uint8_t* ptr = (uint8_t *)sptr;
|
@@ -222,12 +225,12 @@ the_end:
|
|
222
225
|
* input. Of course, it doesn't hurt if the hash function is reasonably fast
|
223
226
|
* either, as it gets called a lot.
|
224
227
|
*/
|
225
|
-
static
|
228
|
+
static INLINE uint32_t HashBytes(uint32_t bytes, int shift)
|
226
229
|
{
|
227
230
|
uint32_t kMul = 0x1e35a7bd;
|
228
231
|
return (bytes * kMul) >> shift;
|
229
232
|
}
|
230
|
-
static
|
233
|
+
static INLINE uint32_t Hash(const char *p, int shift)
|
231
234
|
{
|
232
235
|
return HashBytes(UNALIGNED_LOAD32(p), shift);
|
233
236
|
}
|
@@ -247,7 +250,7 @@ static inline uint32_t Hash(const char *p, int shift)
|
|
247
250
|
* x86_64 is little endian.
|
248
251
|
*/
|
249
252
|
#if defined(__x86_64__)
|
250
|
-
static
|
253
|
+
static INLINE int
|
251
254
|
FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
252
255
|
{
|
253
256
|
uint64_t x;
|
@@ -291,7 +294,7 @@ FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
|
291
294
|
return matched;
|
292
295
|
}
|
293
296
|
#else /* !defined(__x86_64__) */
|
294
|
-
static
|
297
|
+
static INLINE int
|
295
298
|
FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
296
299
|
{
|
297
300
|
/* Implementation based on the x86-64 version, above. */
|
@@ -326,7 +329,7 @@ FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
|
326
329
|
#endif /* !defined(__x86_64__) */
|
327
330
|
|
328
331
|
|
329
|
-
static
|
332
|
+
static INLINE char*
|
330
333
|
EmitLiteral(char *op, const char *literal, int len, int allow_fast_path)
|
331
334
|
{
|
332
335
|
int n = len - 1; /* Zero-length literals are disallowed */
|
@@ -367,7 +370,7 @@ EmitLiteral(char *op, const char *literal, int len, int allow_fast_path)
|
|
367
370
|
return op + len;
|
368
371
|
}
|
369
372
|
|
370
|
-
static
|
373
|
+
static INLINE char*
|
371
374
|
EmitCopyLessThan64(char *op, int offset, int len)
|
372
375
|
{
|
373
376
|
DCHECK_LE(len, 64);
|
@@ -377,19 +380,19 @@ EmitCopyLessThan64(char *op, int offset, int len)
|
|
377
380
|
if ((len < 12) && (offset < 2048)) {
|
378
381
|
int len_minus_4 = len - 4;
|
379
382
|
DCHECK_LT(len_minus_4, 8); /* Must fit in 3 bits */
|
380
|
-
*op++ = COPY_1_BYTE_OFFSET
|
381
|
-
((len_minus_4) << 2)
|
383
|
+
*op++ = COPY_1_BYTE_OFFSET +
|
384
|
+
((len_minus_4) << 2) +
|
382
385
|
((offset >> 8) << 5);
|
383
386
|
*op++ = offset & 0xff;
|
384
387
|
} else {
|
385
|
-
*op++ = COPY_2_BYTE_OFFSET
|
388
|
+
*op++ = COPY_2_BYTE_OFFSET + ((len-1) << 2);
|
386
389
|
put_unaligned_le16(offset, op);
|
387
390
|
op += 2;
|
388
391
|
}
|
389
392
|
return op;
|
390
393
|
}
|
391
394
|
|
392
|
-
static
|
395
|
+
static INLINE char*
|
393
396
|
EmitCopy(char *op, int offset, int len)
|
394
397
|
{
|
395
398
|
/* Emit 64 byte copies but make sure to keep at least four bytes
|
@@ -420,7 +423,7 @@ empirically found that overlapping loads such as
|
|
420
423
|
are slower than UNALIGNED_LOAD64(p) followed by shifts and casts to uint32.
|
421
424
|
|
422
425
|
We have different versions for 64- and 32-bit; ideally we would avoid the
|
423
|
-
two functions and just
|
426
|
+
two functions and just INLINE the UNALIGNED_LOAD64 call into
|
424
427
|
GetUint32AtOffset, but GCC (at least not as of 4.6) is seemingly not clever
|
425
428
|
enough to avoid loading the value multiple times then. For 64-bit, the load
|
426
429
|
is done when GetEightBytesAt() is called, whereas for 32-bit, the load is
|
@@ -431,11 +434,11 @@ done at GetUint32AtOffset() time.
|
|
431
434
|
|
432
435
|
typedef uint64_t EightBytesReference;
|
433
436
|
|
434
|
-
static
|
437
|
+
static INLINE EightBytesReference GetEightBytesAt(const char* ptr) {
|
435
438
|
return UNALIGNED_LOAD64(ptr);
|
436
439
|
}
|
437
440
|
|
438
|
-
static
|
441
|
+
static INLINE uint32_t GetUint32AtOffset(uint64_t v, int offset) {
|
439
442
|
DCHECK_GE(offset, 0);
|
440
443
|
DCHECK_LE(offset, 4);
|
441
444
|
#ifdef __LITTLE_ENDIAN
|
@@ -449,11 +452,11 @@ static inline uint32_t GetUint32AtOffset(uint64_t v, int offset) {
|
|
449
452
|
|
450
453
|
typedef const char* EightBytesReference;
|
451
454
|
|
452
|
-
static
|
455
|
+
static INLINE EightBytesReference GetEightBytesAt(const char* ptr) {
|
453
456
|
return ptr;
|
454
457
|
}
|
455
458
|
|
456
|
-
static
|
459
|
+
static INLINE uint32_t GetUint32AtOffset(const char* v, int offset) {
|
457
460
|
DCHECK_GE(offset, 0);
|
458
461
|
DCHECK_LE(offset, 4);
|
459
462
|
return UNALIGNED_LOAD32(v + offset);
|
@@ -30,6 +30,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30
30
|
|
31
31
|
File modified for the Linux Kernel by
|
32
32
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
33
|
+
|
34
|
+
File modified for Sereal by
|
35
|
+
Steffen Mueller <smueller@cpan.org>
|
33
36
|
*/
|
34
37
|
|
35
38
|
#include "csnappy_internal.h"
|
@@ -194,7 +197,7 @@ static const uint16_t char_table[256] = {
|
|
194
197
|
* Note that this does not match the semantics of either memcpy()
|
195
198
|
* or memmove().
|
196
199
|
*/
|
197
|
-
static
|
200
|
+
static INLINE void IncrementalCopy(const char *src, char *op, int len)
|
198
201
|
{
|
199
202
|
DCHECK_GT(len, 0);
|
200
203
|
do {
|
@@ -235,7 +238,7 @@ static inline void IncrementalCopy(const char *src, char *op, int len)
|
|
235
238
|
* position 1. Thus, ten excess bytes.
|
236
239
|
*/
|
237
240
|
static const int kMaxIncrementCopyOverflow = 10;
|
238
|
-
static
|
241
|
+
static INLINE void IncrementalCopyFastPath(const char *src, char *op, int len)
|
239
242
|
{
|
240
243
|
while (op - src < 8) {
|
241
244
|
UnalignedCopy64(src, op);
|
@@ -258,7 +261,7 @@ struct SnappyArrayWriter {
|
|
258
261
|
char *op_limit;
|
259
262
|
};
|
260
263
|
|
261
|
-
static
|
264
|
+
static INLINE int
|
262
265
|
SAW__AppendFastPath(struct SnappyArrayWriter *this,
|
263
266
|
const char *ip, uint32_t len)
|
264
267
|
{
|
@@ -276,7 +279,7 @@ SAW__AppendFastPath(struct SnappyArrayWriter *this,
|
|
276
279
|
return CSNAPPY_E_OK;
|
277
280
|
}
|
278
281
|
|
279
|
-
static
|
282
|
+
static INLINE int
|
280
283
|
SAW__Append(struct SnappyArrayWriter *this,
|
281
284
|
const char *ip, uint32_t len)
|
282
285
|
{
|
@@ -289,7 +292,7 @@ SAW__Append(struct SnappyArrayWriter *this,
|
|
289
292
|
return CSNAPPY_E_OK;
|
290
293
|
}
|
291
294
|
|
292
|
-
static
|
295
|
+
static INLINE int
|
293
296
|
SAW__AppendFromSelf(struct SnappyArrayWriter *this,
|
294
297
|
uint32_t offset, uint32_t len)
|
295
298
|
{
|
@@ -31,11 +31,16 @@ Various stubs for the open-source version of Snappy.
|
|
31
31
|
|
32
32
|
File modified for the Linux Kernel by
|
33
33
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
34
|
+
|
35
|
+
File modified for Sereal by
|
36
|
+
Steffen Mueller <smueller@cpan.org>
|
34
37
|
*/
|
35
38
|
|
36
39
|
#ifndef CSNAPPY_INTERNAL_H_
|
37
40
|
#define CSNAPPY_INTERNAL_H_
|
38
41
|
|
42
|
+
#include "csnappy_compat.h"
|
43
|
+
|
39
44
|
#ifndef __KERNEL__
|
40
45
|
#include "csnappy_internal_userspace.h"
|
41
46
|
#include <string.h>
|
@@ -77,11 +82,15 @@ Zeev Tarantov <zeev.tarantov@gmail.com>
|
|
77
82
|
|
78
83
|
#endif /* __KERNEL__ */
|
79
84
|
|
85
|
+
#if (!defined(__LITTLE_ENDIAN) && !defined(__BIG_ENDIAN)) || ! defined(__BYTE_ORDER)
|
86
|
+
# error either __LITTLE_ENDIAN or __BIG_ENDIAN, plus __BYTE_ORDER must be defined
|
87
|
+
#endif
|
88
|
+
|
80
89
|
#define ARCH_ARM_HAVE_UNALIGNED \
|
81
90
|
defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) || defined(__ARMV6__) || \
|
82
91
|
defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__)
|
83
92
|
|
84
|
-
static
|
93
|
+
static INLINE void UnalignedCopy64(const void *src, void *dst) {
|
85
94
|
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || ARCH_ARM_HAVE_UNALIGNED
|
86
95
|
if ((sizeof(void *) == 8) || (sizeof(long) == 8)) {
|
87
96
|
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
|
@@ -110,7 +119,7 @@ static inline void UnalignedCopy64(const void *src, void *dst) {
|
|
110
119
|
|
111
120
|
#if defined(__arm__)
|
112
121
|
#if ARCH_ARM_HAVE_UNALIGNED
|
113
|
-
static
|
122
|
+
static INLINE uint32_t get_unaligned_le(const void *p, uint32_t n)
|
114
123
|
{
|
115
124
|
uint32_t wordmask = (1U << (8 * n)) - 1;
|
116
125
|
return get_unaligned_le32(p) & wordmask;
|
@@ -120,7 +129,7 @@ static inline void UnalignedCopy64(const void *src, void *dst) {
|
|
120
129
|
#define get_unaligned_le get_unaligned_le_armv5
|
121
130
|
#endif
|
122
131
|
#else
|
123
|
-
static
|
132
|
+
static INLINE uint32_t get_unaligned_le(const void *p, uint32_t n)
|
124
133
|
{
|
125
134
|
/* Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits */
|
126
135
|
static const uint32_t wordmask[] = {
|
@@ -31,16 +31,43 @@ Various stubs for the open-source version of Snappy.
|
|
31
31
|
|
32
32
|
File modified by
|
33
33
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
34
|
+
|
35
|
+
File modified for Sereal by
|
36
|
+
Steffen Mueller <smueller@cpan.org>
|
34
37
|
*/
|
35
38
|
|
36
39
|
#ifndef CSNAPPY_INTERNAL_USERSPACE_H_
|
37
40
|
#define CSNAPPY_INTERNAL_USERSPACE_H_
|
38
41
|
|
39
|
-
|
42
|
+
/*note the original version of this file checked for MS version, but MS will *never* support
|
43
|
+
* anything but C89, so the version check is bogus. */
|
44
|
+
#if defined(_MSC_VER)
|
40
45
|
typedef unsigned __int8 uint8_t;
|
41
46
|
typedef unsigned __int16 uint16_t;
|
42
47
|
typedef unsigned __int32 uint32_t;
|
43
48
|
typedef unsigned __int64 uint64_t;
|
49
|
+
typedef __int32 int32_t; /* Sereal specific change, see csnappy_decompress.c(271) : error C2065: 'int32_t' : undeclared identifier */
|
50
|
+
/* the following define is Sereal specific, as MS C89 compilers do not know about "inline" */
|
51
|
+
#define inline __inline
|
52
|
+
#ifdef _M_X64
|
53
|
+
# define __x86_64__
|
54
|
+
# define __x86_64
|
55
|
+
# define __amd64__
|
56
|
+
# define __amd64
|
57
|
+
#endif
|
58
|
+
#ifdef _M_IX86
|
59
|
+
# define __i386__
|
60
|
+
# define __i386
|
61
|
+
# define i386
|
62
|
+
# define _X86_
|
63
|
+
#endif
|
64
|
+
#ifdef _M_IA64
|
65
|
+
# define __ia64__
|
66
|
+
# define __ia64
|
67
|
+
# define __IA64__
|
68
|
+
# define __itanium__
|
69
|
+
#endif
|
70
|
+
|
44
71
|
#else
|
45
72
|
#include <stdint.h>
|
46
73
|
#endif
|
@@ -70,6 +97,8 @@ typedef unsigned __int64 uint64_t;
|
|
70
97
|
#define DCHECK(cond)
|
71
98
|
#endif
|
72
99
|
|
100
|
+
#include "csnappy_compat.h"
|
101
|
+
|
73
102
|
/*
|
74
103
|
Uses code from http://code.google.com/p/exfat/source/browse/trunk/libexfat/byteorder.h
|
75
104
|
with 3-clause BSD license instead of GPL, with permission from:
|
@@ -82,6 +111,9 @@ Albert Lee
|
|
82
111
|
#define bswap_16(x) _byteswap_ushort(x)
|
83
112
|
#define bswap_32(x) _byteswap_ulong(x)
|
84
113
|
#define bswap_64(x) _byteswap_uint64(x)
|
114
|
+
#define __BIG_ENDIAN 4321
|
115
|
+
#define __LITTLE_ENDIAN 1234
|
116
|
+
#define __BYTE_ORDER LITTLE_ENDIAN
|
85
117
|
|
86
118
|
#elif defined(__GLIBC__) || defined(__ANDROID__) || defined(__CYGWIN__)
|
87
119
|
|
@@ -99,7 +131,7 @@ Albert Lee
|
|
99
131
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
100
132
|
#define __BIG_ENDIAN BIG_ENDIAN
|
101
133
|
|
102
|
-
#elif defined(__FreeBSD__) || defined(
|
134
|
+
#elif defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__)
|
103
135
|
|
104
136
|
#include <sys/endian.h>
|
105
137
|
#define bswap_16(x) bswap16(x)
|
@@ -133,6 +165,12 @@ Albert Lee
|
|
133
165
|
#define __BYTE_ORDER __BIG_ENDIAN
|
134
166
|
#endif
|
135
167
|
|
168
|
+
#elif defined(__MINGW32__)
|
169
|
+
#include <sys/param.h>
|
170
|
+
#define __BYTE_ORDER BYTE_ORDER
|
171
|
+
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
172
|
+
#define __BIG_ENDIAN BIG_ENDIAN
|
173
|
+
|
136
174
|
#endif
|
137
175
|
|
138
176
|
|
@@ -169,13 +207,13 @@ Albert Lee
|
|
169
207
|
struct una_u64 { uint64_t x; };
|
170
208
|
#pragma pack()
|
171
209
|
|
172
|
-
static
|
210
|
+
static INLINE uint64_t UNALIGNED_LOAD64(const void *p)
|
173
211
|
{
|
174
212
|
const struct una_u64 *ptr = (const struct una_u64 *)p;
|
175
213
|
return ptr->x;
|
176
214
|
}
|
177
215
|
|
178
|
-
static
|
216
|
+
static INLINE void UNALIGNED_STORE64(void *p, uint64_t v)
|
179
217
|
{
|
180
218
|
struct una_u64 *ptr = (struct una_u64 *)p;
|
181
219
|
ptr->x = v;
|
@@ -189,37 +227,37 @@ struct una_u32 { uint32_t x; };
|
|
189
227
|
struct una_u64 { uint64_t x; };
|
190
228
|
#pragma pack()
|
191
229
|
|
192
|
-
static
|
230
|
+
static INLINE uint16_t UNALIGNED_LOAD16(const void *p)
|
193
231
|
{
|
194
232
|
const struct una_u16 *ptr = (const struct una_u16 *)p;
|
195
233
|
return ptr->x;
|
196
234
|
}
|
197
235
|
|
198
|
-
static
|
236
|
+
static INLINE uint32_t UNALIGNED_LOAD32(const void *p)
|
199
237
|
{
|
200
238
|
const struct una_u32 *ptr = (const struct una_u32 *)p;
|
201
239
|
return ptr->x;
|
202
240
|
}
|
203
241
|
|
204
|
-
static
|
242
|
+
static INLINE uint64_t UNALIGNED_LOAD64(const void *p)
|
205
243
|
{
|
206
244
|
const struct una_u64 *ptr = (const struct una_u64 *)p;
|
207
245
|
return ptr->x;
|
208
246
|
}
|
209
247
|
|
210
|
-
static
|
248
|
+
static INLINE void UNALIGNED_STORE16(void *p, uint16_t v)
|
211
249
|
{
|
212
250
|
struct una_u16 *ptr = (struct una_u16 *)p;
|
213
251
|
ptr->x = v;
|
214
252
|
}
|
215
253
|
|
216
|
-
static
|
254
|
+
static INLINE void UNALIGNED_STORE32(void *p, uint32_t v)
|
217
255
|
{
|
218
256
|
struct una_u32 *ptr = (struct una_u32 *)p;
|
219
257
|
ptr->x = v;
|
220
258
|
}
|
221
259
|
|
222
|
-
static
|
260
|
+
static INLINE void UNALIGNED_STORE64(void *p, uint64_t v)
|
223
261
|
{
|
224
262
|
struct una_u64 *ptr = (struct una_u64 *)p;
|
225
263
|
ptr->x = v;
|
@@ -232,21 +270,21 @@ static inline void UNALIGNED_STORE64(void *p, uint64_t v)
|
|
232
270
|
#define get_unaligned_le32(p) UNALIGNED_LOAD32(p)
|
233
271
|
#define put_unaligned_le16(v, p) UNALIGNED_STORE16(p, v)
|
234
272
|
#elif __BYTE_ORDER == __BIG_ENDIAN
|
235
|
-
static
|
273
|
+
static INLINE uint32_t get_unaligned_le32(const void *p)
|
236
274
|
{
|
237
275
|
return bswap_32(UNALIGNED_LOAD32(p));
|
238
276
|
}
|
239
|
-
static
|
277
|
+
static INLINE void put_unaligned_le16(uint16_t val, void *p)
|
240
278
|
{
|
241
279
|
UNALIGNED_STORE16(p, bswap_16(val));
|
242
280
|
}
|
243
281
|
#else
|
244
|
-
static
|
282
|
+
static INLINE uint32_t get_unaligned_le32(const void *p)
|
245
283
|
{
|
246
284
|
const uint8_t *b = (const uint8_t *)p;
|
247
285
|
return b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24);
|
248
286
|
}
|
249
|
-
static
|
287
|
+
static INLINE void put_unaligned_le16(uint16_t val, void *p)
|
250
288
|
{
|
251
289
|
uint8_t *b = (uint8_t *)p;
|
252
290
|
b[0] = val & 255;
|
@@ -257,19 +295,19 @@ static inline void put_unaligned_le16(uint16_t val, void *p)
|
|
257
295
|
|
258
296
|
#if defined(HAVE_BUILTIN_CTZ)
|
259
297
|
|
260
|
-
static
|
298
|
+
static INLINE int FindLSBSetNonZero(uint32_t n)
|
261
299
|
{
|
262
300
|
return __builtin_ctz(n);
|
263
301
|
}
|
264
302
|
|
265
|
-
static
|
303
|
+
static INLINE int FindLSBSetNonZero64(uint64_t n)
|
266
304
|
{
|
267
305
|
return __builtin_ctzll(n);
|
268
306
|
}
|
269
307
|
|
270
308
|
#else /* Portable versions. */
|
271
309
|
|
272
|
-
static
|
310
|
+
static INLINE int FindLSBSetNonZero(uint32_t n)
|
273
311
|
{
|
274
312
|
int rc = 31, i, shift;
|
275
313
|
uint32_t x;
|
@@ -285,7 +323,7 @@ static inline int FindLSBSetNonZero(uint32_t n)
|
|
285
323
|
}
|
286
324
|
|
287
325
|
/* FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero(). */
|
288
|
-
static
|
326
|
+
static INLINE int FindLSBSetNonZero64(uint64_t n)
|
289
327
|
{
|
290
328
|
const uint32_t bottombits = (uint32_t)n;
|
291
329
|
if (bottombits == 0) {
|