sereal 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/sereal/buffer.h +15 -8
- data/ext/sereal/decode.c +72 -28
- data/ext/sereal/encode.c +107 -18
- data/ext/sereal/lz4.c +822 -0
- data/ext/sereal/lz4.h +205 -0
- data/ext/sereal/lz4/lz4.c +822 -0
- data/ext/sereal/lz4/lz4.h +205 -0
- data/ext/sereal/lz4/lz4hc.c +817 -0
- data/ext/sereal/lz4/lz4hc.h +111 -0
- data/ext/sereal/lz4hc.c +817 -0
- data/ext/sereal/lz4hc.h +111 -0
- data/ext/sereal/proto.h +4 -2
- data/ext/sereal/sereal.c +30 -0
- data/ext/sereal/sereal.h +13 -1
- data/ext/sereal/snappy/csnappy_compat.h +16 -0
- data/ext/sereal/snappy/csnappy_compress.c +19 -16
- data/ext/sereal/snappy/csnappy_decompress.c +8 -5
- data/ext/sereal/snappy/csnappy_internal.h +12 -3
- data/ext/sereal/snappy/csnappy_internal_userspace.h +56 -18
- metadata +11 -2
data/ext/sereal/lz4hc.h
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
/*
|
2
|
+
LZ4 HC - High Compression Mode of LZ4
|
3
|
+
Header File
|
4
|
+
Copyright (C) 2011-2013, Yann Collet.
|
5
|
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
6
|
+
|
7
|
+
Redistribution and use in source and binary forms, with or without
|
8
|
+
modification, are permitted provided that the following conditions are
|
9
|
+
met:
|
10
|
+
|
11
|
+
* Redistributions of source code must retain the above copyright
|
12
|
+
notice, this list of conditions and the following disclaimer.
|
13
|
+
* Redistributions in binary form must reproduce the above
|
14
|
+
copyright notice, this list of conditions and the following disclaimer
|
15
|
+
in the documentation and/or other materials provided with the
|
16
|
+
distribution.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
+
|
30
|
+
You can contact the author at :
|
31
|
+
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
|
32
|
+
- LZ4 source repository : http://code.google.com/p/lz4/
|
33
|
+
*/
|
34
|
+
#pragma once
|
35
|
+
|
36
|
+
|
37
|
+
#if defined (__cplusplus)
|
38
|
+
extern "C" {
|
39
|
+
#endif
|
40
|
+
|
41
|
+
|
42
|
+
int LZ4_compressHC (const char* source, char* dest, int inputSize);
|
43
|
+
/*
|
44
|
+
LZ4_compressHC :
|
45
|
+
return : the number of bytes in compressed buffer dest
|
46
|
+
or 0 if compression fails.
|
47
|
+
note : destination buffer must be already allocated.
|
48
|
+
To avoid any problem, size it to handle worst cases situations (input data not compressible)
|
49
|
+
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
|
50
|
+
*/
|
51
|
+
|
52
|
+
int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
|
53
|
+
/*
|
54
|
+
LZ4_compress_limitedOutput() :
|
55
|
+
Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
|
56
|
+
If it cannot achieve it, compression will stop, and result of the function will be zero.
|
57
|
+
This function never writes outside of provided output buffer.
|
58
|
+
|
59
|
+
inputSize : Max supported value is 1 GB
|
60
|
+
maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
|
61
|
+
return : the number of output bytes written in buffer 'dest'
|
62
|
+
or 0 if compression fails.
|
63
|
+
*/
|
64
|
+
|
65
|
+
|
66
|
+
/* Note :
|
67
|
+
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
|
68
|
+
*/
|
69
|
+
|
70
|
+
|
71
|
+
/* Advanced Functions */
|
72
|
+
|
73
|
+
void* LZ4_createHC (const char* inputBuffer);
|
74
|
+
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
|
75
|
+
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
|
76
|
+
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
|
77
|
+
int LZ4_freeHC (void* LZ4HC_Data);
|
78
|
+
|
79
|
+
/*
|
80
|
+
These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
|
81
|
+
In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
|
82
|
+
|
83
|
+
void* LZ4_createHC (const char* inputBuffer);
|
84
|
+
The result of the function is the (void*) pointer on the LZ4HC Data Structure.
|
85
|
+
This pointer will be needed in all other functions.
|
86
|
+
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
|
87
|
+
The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
|
88
|
+
The input buffer must be already allocated, and size at least 192KB.
|
89
|
+
'inputBuffer' will also be the 'const char* source' of the first block.
|
90
|
+
|
91
|
+
All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
|
92
|
+
To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
|
93
|
+
Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
|
94
|
+
but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
|
95
|
+
If next block does not begin immediately after the previous one, the compression will fail (return 0).
|
96
|
+
|
97
|
+
When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
|
98
|
+
char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
|
99
|
+
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
|
100
|
+
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
|
101
|
+
==> The memory position where the next input data block must start is provided as the result of the function.
|
102
|
+
|
103
|
+
Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
|
104
|
+
|
105
|
+
When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
|
106
|
+
*/
|
107
|
+
|
108
|
+
|
109
|
+
#if defined (__cplusplus)
|
110
|
+
}
|
111
|
+
#endif
|
data/ext/sereal/proto.h
CHANGED
@@ -6,8 +6,10 @@
|
|
6
6
|
#define SRL_PROTOCOL_VERSION_MASK ( ( 1 << SRL_PROTOCOL_VERSION_BITS ) - 1 )
|
7
7
|
|
8
8
|
#define SRL_PROTOCOL_ENCODING_MASK ( ~SRL_PROTOCOL_VERSION_MASK )
|
9
|
-
#define SRL_PROTOCOL_ENCODING_RAW
|
10
|
-
#define SRL_PROTOCOL_ENCODING_SNAPPY
|
9
|
+
#define SRL_PROTOCOL_ENCODING_RAW ( 0 << SRL_PROTOCOL_VERSION_BITS )
|
10
|
+
#define SRL_PROTOCOL_ENCODING_SNAPPY ( 1 << SRL_PROTOCOL_VERSION_BITS )
|
11
|
+
#define SRL_PROTOCOL_ENCODING_SNAPPY_INCR ( 2 << SRL_PROTOCOL_VERSION_BITS )
|
12
|
+
#define SRL_PROTOCOL_ENCODING_LZ4_INCR ( 3 << SRL_PROTOCOL_VERSION_BITS )
|
11
13
|
|
12
14
|
#define SRL_HDR_POS ((char)0) /* small positive integer - value in low 4 bits (identity) */
|
13
15
|
#define SRL_HDR_POS_LOW ((char)0) /* small positive integer - value in low 4 bits (identity) */
|
data/ext/sereal/sereal.c
CHANGED
@@ -3,10 +3,40 @@
|
|
3
3
|
|
4
4
|
VALUE Sereal = Qnil;
|
5
5
|
void Init_sereal();
|
6
|
+
|
7
|
+
/*
|
8
|
+
* Encode/Decode object using Sereal binary protocol:
|
9
|
+
* https://github.com/Sereal/Sereal/blob/master/sereal_spec.pod
|
10
|
+
*
|
11
|
+
* Sereal.encode(object) -> serialized blob
|
12
|
+
* Sereal.encode(object,Sereal::LZ4) -> LZ4 compressed blob
|
13
|
+
* Sereal.encode(object,Sereal::LZ4HC) -> LZ4HC compressed blob
|
14
|
+
* Sereal.encode(object,Sereal::SNAPPY_INCR) -> snappy compressed blob
|
15
|
+
* Sereal.encode(object,Sereal::SNAPPY) -> snappy compressed blob
|
16
|
+
*
|
17
|
+
* LZ4 LZ4HC and SNAPPY_INCR can be appended into one output and then the
|
18
|
+
* decoder will know what to do.
|
19
|
+
*
|
20
|
+
* Sereal.decode(blob) - returns the decoded object
|
21
|
+
*
|
22
|
+
* If the blob contains multiple compressed(with LZ4* or SNAPPY_INCR)
|
23
|
+
* sub-blobs you should call it with:
|
24
|
+
*
|
25
|
+
* Sereal.decode(blob) do |decoded|
|
26
|
+
* # do something with the decoded object
|
27
|
+
* end
|
28
|
+
* otherwise only the first decoded object will be returned
|
29
|
+
*
|
30
|
+
*/
|
6
31
|
void Init_sereal() {
|
7
32
|
Sereal = rb_define_class("Sereal", rb_cObject);
|
8
33
|
rb_define_singleton_method(Sereal, "encode", method_sereal_encode, -2);
|
9
34
|
rb_define_singleton_method(Sereal, "decode", method_sereal_decode, -2);
|
35
|
+
rb_define_const(Sereal, "SNAPPY",INT2NUM(__SNAPPY));
|
36
|
+
rb_define_const(Sereal, "SNAPPY_INCR",INT2NUM(__SNAPPY_INCR));
|
37
|
+
rb_define_const(Sereal, "RAW",INT2NUM(__RAW));
|
38
|
+
rb_define_const(Sereal, "LZ4",INT2NUM(__LZ4_INCR));
|
39
|
+
rb_define_const(Sereal, "LZ4HC",INT2NUM(__LZ4HC_INCR));
|
10
40
|
s_init_writers();
|
11
41
|
}
|
12
42
|
|
data/ext/sereal/sereal.h
CHANGED
@@ -40,7 +40,13 @@ typedef struct _track_entry track_t;
|
|
40
40
|
#define MULTILINE RE_OPTION_MULTILINE
|
41
41
|
#define EXTENDED RE_OPTION_EXTENDED
|
42
42
|
#endif
|
43
|
-
#define
|
43
|
+
#define s_raise(what,ex,arg...) \
|
44
|
+
do { \
|
45
|
+
s_destroy(what); \
|
46
|
+
rb_raise(ex,##arg); \
|
47
|
+
} while(0);
|
48
|
+
|
49
|
+
#define FLAG_NOT_MINE 1
|
44
50
|
struct _sereal {
|
45
51
|
u8 *data;
|
46
52
|
u32 size;
|
@@ -70,4 +76,10 @@ do { \
|
|
70
76
|
#define rb_sym_to_s(object) rb_funcall(object,rb_intern("to_s"),0)
|
71
77
|
#endif
|
72
78
|
|
79
|
+
#define __RAW 0
|
80
|
+
#define __SNAPPY 1
|
81
|
+
#define __SNAPPY_INCR 2
|
82
|
+
#define __LZ4_INCR 3
|
83
|
+
#define __LZ4HC_INCR 4
|
84
|
+
|
73
85
|
#endif
|
@@ -0,0 +1,16 @@
|
|
1
|
+
#ifndef CSNAPPY_COMPAT_H
|
2
|
+
|
3
|
+
/* This file was added to Sereal to attempt some MSVC compatibility,
|
4
|
+
* but is at best a band-aid. And done without a lot of experience
|
5
|
+
* in whatever subset of C99 MSVC supports.
|
6
|
+
*/
|
7
|
+
|
8
|
+
#ifndef INLINE
|
9
|
+
# if defined(_MSC_VER)
|
10
|
+
# define INLINE __inline
|
11
|
+
# else
|
12
|
+
# define INLINE inline
|
13
|
+
# endif
|
14
|
+
#endif
|
15
|
+
|
16
|
+
#endif
|
@@ -30,6 +30,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30
30
|
|
31
31
|
File modified for the Linux Kernel by
|
32
32
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
33
|
+
|
34
|
+
File modified for Sereal by
|
35
|
+
Steffen Mueller <smueller@cpan.org>
|
33
36
|
*/
|
34
37
|
|
35
38
|
#include "csnappy_internal.h"
|
@@ -40,7 +43,7 @@ Zeev Tarantov <zeev.tarantov@gmail.com>
|
|
40
43
|
#include "csnappy.h"
|
41
44
|
|
42
45
|
|
43
|
-
static
|
46
|
+
static INLINE char*
|
44
47
|
encode_varint32(char *sptr, uint32_t v)
|
45
48
|
{
|
46
49
|
uint8_t* ptr = (uint8_t *)sptr;
|
@@ -222,12 +225,12 @@ the_end:
|
|
222
225
|
* input. Of course, it doesn't hurt if the hash function is reasonably fast
|
223
226
|
* either, as it gets called a lot.
|
224
227
|
*/
|
225
|
-
static
|
228
|
+
static INLINE uint32_t HashBytes(uint32_t bytes, int shift)
|
226
229
|
{
|
227
230
|
uint32_t kMul = 0x1e35a7bd;
|
228
231
|
return (bytes * kMul) >> shift;
|
229
232
|
}
|
230
|
-
static
|
233
|
+
static INLINE uint32_t Hash(const char *p, int shift)
|
231
234
|
{
|
232
235
|
return HashBytes(UNALIGNED_LOAD32(p), shift);
|
233
236
|
}
|
@@ -247,7 +250,7 @@ static inline uint32_t Hash(const char *p, int shift)
|
|
247
250
|
* x86_64 is little endian.
|
248
251
|
*/
|
249
252
|
#if defined(__x86_64__)
|
250
|
-
static
|
253
|
+
static INLINE int
|
251
254
|
FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
252
255
|
{
|
253
256
|
uint64_t x;
|
@@ -291,7 +294,7 @@ FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
|
291
294
|
return matched;
|
292
295
|
}
|
293
296
|
#else /* !defined(__x86_64__) */
|
294
|
-
static
|
297
|
+
static INLINE int
|
295
298
|
FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
296
299
|
{
|
297
300
|
/* Implementation based on the x86-64 version, above. */
|
@@ -326,7 +329,7 @@ FindMatchLength(const char *s1, const char *s2, const char *s2_limit)
|
|
326
329
|
#endif /* !defined(__x86_64__) */
|
327
330
|
|
328
331
|
|
329
|
-
static
|
332
|
+
static INLINE char*
|
330
333
|
EmitLiteral(char *op, const char *literal, int len, int allow_fast_path)
|
331
334
|
{
|
332
335
|
int n = len - 1; /* Zero-length literals are disallowed */
|
@@ -367,7 +370,7 @@ EmitLiteral(char *op, const char *literal, int len, int allow_fast_path)
|
|
367
370
|
return op + len;
|
368
371
|
}
|
369
372
|
|
370
|
-
static
|
373
|
+
static INLINE char*
|
371
374
|
EmitCopyLessThan64(char *op, int offset, int len)
|
372
375
|
{
|
373
376
|
DCHECK_LE(len, 64);
|
@@ -377,19 +380,19 @@ EmitCopyLessThan64(char *op, int offset, int len)
|
|
377
380
|
if ((len < 12) && (offset < 2048)) {
|
378
381
|
int len_minus_4 = len - 4;
|
379
382
|
DCHECK_LT(len_minus_4, 8); /* Must fit in 3 bits */
|
380
|
-
*op++ = COPY_1_BYTE_OFFSET
|
381
|
-
((len_minus_4) << 2)
|
383
|
+
*op++ = COPY_1_BYTE_OFFSET +
|
384
|
+
((len_minus_4) << 2) +
|
382
385
|
((offset >> 8) << 5);
|
383
386
|
*op++ = offset & 0xff;
|
384
387
|
} else {
|
385
|
-
*op++ = COPY_2_BYTE_OFFSET
|
388
|
+
*op++ = COPY_2_BYTE_OFFSET + ((len-1) << 2);
|
386
389
|
put_unaligned_le16(offset, op);
|
387
390
|
op += 2;
|
388
391
|
}
|
389
392
|
return op;
|
390
393
|
}
|
391
394
|
|
392
|
-
static
|
395
|
+
static INLINE char*
|
393
396
|
EmitCopy(char *op, int offset, int len)
|
394
397
|
{
|
395
398
|
/* Emit 64 byte copies but make sure to keep at least four bytes
|
@@ -420,7 +423,7 @@ empirically found that overlapping loads such as
|
|
420
423
|
are slower than UNALIGNED_LOAD64(p) followed by shifts and casts to uint32.
|
421
424
|
|
422
425
|
We have different versions for 64- and 32-bit; ideally we would avoid the
|
423
|
-
two functions and just
|
426
|
+
two functions and just INLINE the UNALIGNED_LOAD64 call into
|
424
427
|
GetUint32AtOffset, but GCC (at least not as of 4.6) is seemingly not clever
|
425
428
|
enough to avoid loading the value multiple times then. For 64-bit, the load
|
426
429
|
is done when GetEightBytesAt() is called, whereas for 32-bit, the load is
|
@@ -431,11 +434,11 @@ done at GetUint32AtOffset() time.
|
|
431
434
|
|
432
435
|
typedef uint64_t EightBytesReference;
|
433
436
|
|
434
|
-
static
|
437
|
+
static INLINE EightBytesReference GetEightBytesAt(const char* ptr) {
|
435
438
|
return UNALIGNED_LOAD64(ptr);
|
436
439
|
}
|
437
440
|
|
438
|
-
static
|
441
|
+
static INLINE uint32_t GetUint32AtOffset(uint64_t v, int offset) {
|
439
442
|
DCHECK_GE(offset, 0);
|
440
443
|
DCHECK_LE(offset, 4);
|
441
444
|
#ifdef __LITTLE_ENDIAN
|
@@ -449,11 +452,11 @@ static inline uint32_t GetUint32AtOffset(uint64_t v, int offset) {
|
|
449
452
|
|
450
453
|
typedef const char* EightBytesReference;
|
451
454
|
|
452
|
-
static
|
455
|
+
static INLINE EightBytesReference GetEightBytesAt(const char* ptr) {
|
453
456
|
return ptr;
|
454
457
|
}
|
455
458
|
|
456
|
-
static
|
459
|
+
static INLINE uint32_t GetUint32AtOffset(const char* v, int offset) {
|
457
460
|
DCHECK_GE(offset, 0);
|
458
461
|
DCHECK_LE(offset, 4);
|
459
462
|
return UNALIGNED_LOAD32(v + offset);
|
@@ -30,6 +30,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30
30
|
|
31
31
|
File modified for the Linux Kernel by
|
32
32
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
33
|
+
|
34
|
+
File modified for Sereal by
|
35
|
+
Steffen Mueller <smueller@cpan.org>
|
33
36
|
*/
|
34
37
|
|
35
38
|
#include "csnappy_internal.h"
|
@@ -194,7 +197,7 @@ static const uint16_t char_table[256] = {
|
|
194
197
|
* Note that this does not match the semantics of either memcpy()
|
195
198
|
* or memmove().
|
196
199
|
*/
|
197
|
-
static
|
200
|
+
static INLINE void IncrementalCopy(const char *src, char *op, int len)
|
198
201
|
{
|
199
202
|
DCHECK_GT(len, 0);
|
200
203
|
do {
|
@@ -235,7 +238,7 @@ static inline void IncrementalCopy(const char *src, char *op, int len)
|
|
235
238
|
* position 1. Thus, ten excess bytes.
|
236
239
|
*/
|
237
240
|
static const int kMaxIncrementCopyOverflow = 10;
|
238
|
-
static
|
241
|
+
static INLINE void IncrementalCopyFastPath(const char *src, char *op, int len)
|
239
242
|
{
|
240
243
|
while (op - src < 8) {
|
241
244
|
UnalignedCopy64(src, op);
|
@@ -258,7 +261,7 @@ struct SnappyArrayWriter {
|
|
258
261
|
char *op_limit;
|
259
262
|
};
|
260
263
|
|
261
|
-
static
|
264
|
+
static INLINE int
|
262
265
|
SAW__AppendFastPath(struct SnappyArrayWriter *this,
|
263
266
|
const char *ip, uint32_t len)
|
264
267
|
{
|
@@ -276,7 +279,7 @@ SAW__AppendFastPath(struct SnappyArrayWriter *this,
|
|
276
279
|
return CSNAPPY_E_OK;
|
277
280
|
}
|
278
281
|
|
279
|
-
static
|
282
|
+
static INLINE int
|
280
283
|
SAW__Append(struct SnappyArrayWriter *this,
|
281
284
|
const char *ip, uint32_t len)
|
282
285
|
{
|
@@ -289,7 +292,7 @@ SAW__Append(struct SnappyArrayWriter *this,
|
|
289
292
|
return CSNAPPY_E_OK;
|
290
293
|
}
|
291
294
|
|
292
|
-
static
|
295
|
+
static INLINE int
|
293
296
|
SAW__AppendFromSelf(struct SnappyArrayWriter *this,
|
294
297
|
uint32_t offset, uint32_t len)
|
295
298
|
{
|
@@ -31,11 +31,16 @@ Various stubs for the open-source version of Snappy.
|
|
31
31
|
|
32
32
|
File modified for the Linux Kernel by
|
33
33
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
34
|
+
|
35
|
+
File modified for Sereal by
|
36
|
+
Steffen Mueller <smueller@cpan.org>
|
34
37
|
*/
|
35
38
|
|
36
39
|
#ifndef CSNAPPY_INTERNAL_H_
|
37
40
|
#define CSNAPPY_INTERNAL_H_
|
38
41
|
|
42
|
+
#include "csnappy_compat.h"
|
43
|
+
|
39
44
|
#ifndef __KERNEL__
|
40
45
|
#include "csnappy_internal_userspace.h"
|
41
46
|
#include <string.h>
|
@@ -77,11 +82,15 @@ Zeev Tarantov <zeev.tarantov@gmail.com>
|
|
77
82
|
|
78
83
|
#endif /* __KERNEL__ */
|
79
84
|
|
85
|
+
#if (!defined(__LITTLE_ENDIAN) && !defined(__BIG_ENDIAN)) || ! defined(__BYTE_ORDER)
|
86
|
+
# error either __LITTLE_ENDIAN or __BIG_ENDIAN, plus __BYTE_ORDER must be defined
|
87
|
+
#endif
|
88
|
+
|
80
89
|
#define ARCH_ARM_HAVE_UNALIGNED \
|
81
90
|
defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) || defined(__ARMV6__) || \
|
82
91
|
defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__)
|
83
92
|
|
84
|
-
static
|
93
|
+
static INLINE void UnalignedCopy64(const void *src, void *dst) {
|
85
94
|
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || ARCH_ARM_HAVE_UNALIGNED
|
86
95
|
if ((sizeof(void *) == 8) || (sizeof(long) == 8)) {
|
87
96
|
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
|
@@ -110,7 +119,7 @@ static inline void UnalignedCopy64(const void *src, void *dst) {
|
|
110
119
|
|
111
120
|
#if defined(__arm__)
|
112
121
|
#if ARCH_ARM_HAVE_UNALIGNED
|
113
|
-
static
|
122
|
+
static INLINE uint32_t get_unaligned_le(const void *p, uint32_t n)
|
114
123
|
{
|
115
124
|
uint32_t wordmask = (1U << (8 * n)) - 1;
|
116
125
|
return get_unaligned_le32(p) & wordmask;
|
@@ -120,7 +129,7 @@ static inline void UnalignedCopy64(const void *src, void *dst) {
|
|
120
129
|
#define get_unaligned_le get_unaligned_le_armv5
|
121
130
|
#endif
|
122
131
|
#else
|
123
|
-
static
|
132
|
+
static INLINE uint32_t get_unaligned_le(const void *p, uint32_t n)
|
124
133
|
{
|
125
134
|
/* Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits */
|
126
135
|
static const uint32_t wordmask[] = {
|
@@ -31,16 +31,43 @@ Various stubs for the open-source version of Snappy.
|
|
31
31
|
|
32
32
|
File modified by
|
33
33
|
Zeev Tarantov <zeev.tarantov@gmail.com>
|
34
|
+
|
35
|
+
File modified for Sereal by
|
36
|
+
Steffen Mueller <smueller@cpan.org>
|
34
37
|
*/
|
35
38
|
|
36
39
|
#ifndef CSNAPPY_INTERNAL_USERSPACE_H_
|
37
40
|
#define CSNAPPY_INTERNAL_USERSPACE_H_
|
38
41
|
|
39
|
-
|
42
|
+
/*note the original version of this file checked for MS version, but MS will *never* support
|
43
|
+
* anything but C89, so the version check is bogus. */
|
44
|
+
#if defined(_MSC_VER)
|
40
45
|
typedef unsigned __int8 uint8_t;
|
41
46
|
typedef unsigned __int16 uint16_t;
|
42
47
|
typedef unsigned __int32 uint32_t;
|
43
48
|
typedef unsigned __int64 uint64_t;
|
49
|
+
typedef __int32 int32_t; /* Sereal specific change, see csnappy_decompress.c(271) : error C2065: 'int32_t' : undeclared identifier */
|
50
|
+
/* the following define is Sereal specific, as MS C89 compilers do not know about "inline" */
|
51
|
+
#define inline __inline
|
52
|
+
#ifdef _M_X64
|
53
|
+
# define __x86_64__
|
54
|
+
# define __x86_64
|
55
|
+
# define __amd64__
|
56
|
+
# define __amd64
|
57
|
+
#endif
|
58
|
+
#ifdef _M_IX86
|
59
|
+
# define __i386__
|
60
|
+
# define __i386
|
61
|
+
# define i386
|
62
|
+
# define _X86_
|
63
|
+
#endif
|
64
|
+
#ifdef _M_IA64
|
65
|
+
# define __ia64__
|
66
|
+
# define __ia64
|
67
|
+
# define __IA64__
|
68
|
+
# define __itanium__
|
69
|
+
#endif
|
70
|
+
|
44
71
|
#else
|
45
72
|
#include <stdint.h>
|
46
73
|
#endif
|
@@ -70,6 +97,8 @@ typedef unsigned __int64 uint64_t;
|
|
70
97
|
#define DCHECK(cond)
|
71
98
|
#endif
|
72
99
|
|
100
|
+
#include "csnappy_compat.h"
|
101
|
+
|
73
102
|
/*
|
74
103
|
Uses code from http://code.google.com/p/exfat/source/browse/trunk/libexfat/byteorder.h
|
75
104
|
with 3-clause BSD license instead of GPL, with permission from:
|
@@ -82,6 +111,9 @@ Albert Lee
|
|
82
111
|
#define bswap_16(x) _byteswap_ushort(x)
|
83
112
|
#define bswap_32(x) _byteswap_ulong(x)
|
84
113
|
#define bswap_64(x) _byteswap_uint64(x)
|
114
|
+
#define __BIG_ENDIAN 4321
|
115
|
+
#define __LITTLE_ENDIAN 1234
|
116
|
+
#define __BYTE_ORDER LITTLE_ENDIAN
|
85
117
|
|
86
118
|
#elif defined(__GLIBC__) || defined(__ANDROID__) || defined(__CYGWIN__)
|
87
119
|
|
@@ -99,7 +131,7 @@ Albert Lee
|
|
99
131
|
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
100
132
|
#define __BIG_ENDIAN BIG_ENDIAN
|
101
133
|
|
102
|
-
#elif defined(__FreeBSD__) || defined(
|
134
|
+
#elif defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__)
|
103
135
|
|
104
136
|
#include <sys/endian.h>
|
105
137
|
#define bswap_16(x) bswap16(x)
|
@@ -133,6 +165,12 @@ Albert Lee
|
|
133
165
|
#define __BYTE_ORDER __BIG_ENDIAN
|
134
166
|
#endif
|
135
167
|
|
168
|
+
#elif defined(__MINGW32__)
|
169
|
+
#include <sys/param.h>
|
170
|
+
#define __BYTE_ORDER BYTE_ORDER
|
171
|
+
#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
172
|
+
#define __BIG_ENDIAN BIG_ENDIAN
|
173
|
+
|
136
174
|
#endif
|
137
175
|
|
138
176
|
|
@@ -169,13 +207,13 @@ Albert Lee
|
|
169
207
|
struct una_u64 { uint64_t x; };
|
170
208
|
#pragma pack()
|
171
209
|
|
172
|
-
static
|
210
|
+
static INLINE uint64_t UNALIGNED_LOAD64(const void *p)
|
173
211
|
{
|
174
212
|
const struct una_u64 *ptr = (const struct una_u64 *)p;
|
175
213
|
return ptr->x;
|
176
214
|
}
|
177
215
|
|
178
|
-
static
|
216
|
+
static INLINE void UNALIGNED_STORE64(void *p, uint64_t v)
|
179
217
|
{
|
180
218
|
struct una_u64 *ptr = (struct una_u64 *)p;
|
181
219
|
ptr->x = v;
|
@@ -189,37 +227,37 @@ struct una_u32 { uint32_t x; };
|
|
189
227
|
struct una_u64 { uint64_t x; };
|
190
228
|
#pragma pack()
|
191
229
|
|
192
|
-
static
|
230
|
+
static INLINE uint16_t UNALIGNED_LOAD16(const void *p)
|
193
231
|
{
|
194
232
|
const struct una_u16 *ptr = (const struct una_u16 *)p;
|
195
233
|
return ptr->x;
|
196
234
|
}
|
197
235
|
|
198
|
-
static
|
236
|
+
static INLINE uint32_t UNALIGNED_LOAD32(const void *p)
|
199
237
|
{
|
200
238
|
const struct una_u32 *ptr = (const struct una_u32 *)p;
|
201
239
|
return ptr->x;
|
202
240
|
}
|
203
241
|
|
204
|
-
static
|
242
|
+
static INLINE uint64_t UNALIGNED_LOAD64(const void *p)
|
205
243
|
{
|
206
244
|
const struct una_u64 *ptr = (const struct una_u64 *)p;
|
207
245
|
return ptr->x;
|
208
246
|
}
|
209
247
|
|
210
|
-
static
|
248
|
+
static INLINE void UNALIGNED_STORE16(void *p, uint16_t v)
|
211
249
|
{
|
212
250
|
struct una_u16 *ptr = (struct una_u16 *)p;
|
213
251
|
ptr->x = v;
|
214
252
|
}
|
215
253
|
|
216
|
-
static
|
254
|
+
static INLINE void UNALIGNED_STORE32(void *p, uint32_t v)
|
217
255
|
{
|
218
256
|
struct una_u32 *ptr = (struct una_u32 *)p;
|
219
257
|
ptr->x = v;
|
220
258
|
}
|
221
259
|
|
222
|
-
static
|
260
|
+
static INLINE void UNALIGNED_STORE64(void *p, uint64_t v)
|
223
261
|
{
|
224
262
|
struct una_u64 *ptr = (struct una_u64 *)p;
|
225
263
|
ptr->x = v;
|
@@ -232,21 +270,21 @@ static inline void UNALIGNED_STORE64(void *p, uint64_t v)
|
|
232
270
|
#define get_unaligned_le32(p) UNALIGNED_LOAD32(p)
|
233
271
|
#define put_unaligned_le16(v, p) UNALIGNED_STORE16(p, v)
|
234
272
|
#elif __BYTE_ORDER == __BIG_ENDIAN
|
235
|
-
static
|
273
|
+
static INLINE uint32_t get_unaligned_le32(const void *p)
|
236
274
|
{
|
237
275
|
return bswap_32(UNALIGNED_LOAD32(p));
|
238
276
|
}
|
239
|
-
static
|
277
|
+
static INLINE void put_unaligned_le16(uint16_t val, void *p)
|
240
278
|
{
|
241
279
|
UNALIGNED_STORE16(p, bswap_16(val));
|
242
280
|
}
|
243
281
|
#else
|
244
|
-
static
|
282
|
+
static INLINE uint32_t get_unaligned_le32(const void *p)
|
245
283
|
{
|
246
284
|
const uint8_t *b = (const uint8_t *)p;
|
247
285
|
return b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24);
|
248
286
|
}
|
249
|
-
static
|
287
|
+
static INLINE void put_unaligned_le16(uint16_t val, void *p)
|
250
288
|
{
|
251
289
|
uint8_t *b = (uint8_t *)p;
|
252
290
|
b[0] = val & 255;
|
@@ -257,19 +295,19 @@ static inline void put_unaligned_le16(uint16_t val, void *p)
|
|
257
295
|
|
258
296
|
#if defined(HAVE_BUILTIN_CTZ)
|
259
297
|
|
260
|
-
static
|
298
|
+
static INLINE int FindLSBSetNonZero(uint32_t n)
|
261
299
|
{
|
262
300
|
return __builtin_ctz(n);
|
263
301
|
}
|
264
302
|
|
265
|
-
static
|
303
|
+
static INLINE int FindLSBSetNonZero64(uint64_t n)
|
266
304
|
{
|
267
305
|
return __builtin_ctzll(n);
|
268
306
|
}
|
269
307
|
|
270
308
|
#else /* Portable versions. */
|
271
309
|
|
272
|
-
static
|
310
|
+
static INLINE int FindLSBSetNonZero(uint32_t n)
|
273
311
|
{
|
274
312
|
int rc = 31, i, shift;
|
275
313
|
uint32_t x;
|
@@ -285,7 +323,7 @@ static inline int FindLSBSetNonZero(uint32_t n)
|
|
285
323
|
}
|
286
324
|
|
287
325
|
/* FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero(). */
|
288
|
-
static
|
326
|
+
static INLINE int FindLSBSetNonZero64(uint64_t n)
|
289
327
|
{
|
290
328
|
const uint32_t bottombits = (uint32_t)n;
|
291
329
|
if (bottombits == 0) {
|