snappy 0.0.17 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.dockerignore +2 -0
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/.gitignore +2 -1
- data/.gitmodules +1 -1
- data/Dockerfile +13 -0
- data/Gemfile +4 -0
- data/README.md +29 -5
- data/Rakefile +32 -29
- data/ext/api.c +6 -1
- data/ext/extconf.rb +23 -16
- data/lib/snappy/hadoop/reader.rb +62 -0
- data/lib/snappy/hadoop/writer.rb +51 -0
- data/lib/snappy/hadoop.rb +22 -0
- data/lib/snappy/reader.rb +14 -10
- data/lib/snappy/shim.rb +1 -1
- data/lib/snappy/version.rb +1 -1
- data/lib/snappy.rb +5 -4
- data/snappy.gemspec +13 -13
- data/test/hadoop/snappy_hadoop_reader_test.rb +115 -0
- data/test/hadoop/snappy_hadoop_writer_test.rb +48 -0
- data/test/snappy_hadoop_test.rb +26 -0
- data/test/snappy_reader_test.rb +148 -0
- data/test/snappy_test.rb +95 -0
- data/test/snappy_writer_test.rb +55 -0
- data/test/test_helper.rb +7 -0
- data/test.sh +3 -0
- data/vendor/snappy/CMakeLists.txt +297 -0
- data/vendor/snappy/CONTRIBUTING.md +26 -0
- data/vendor/snappy/NEWS +40 -0
- data/vendor/snappy/{README → README.md} +27 -18
- data/vendor/snappy/cmake/SnappyConfig.cmake.in +33 -0
- data/vendor/snappy/cmake/config.h.in +62 -0
- data/vendor/snappy/docs/README.md +72 -0
- data/vendor/snappy/snappy-internal.h +22 -18
- data/vendor/snappy/snappy-stubs-internal.cc +1 -1
- data/vendor/snappy/snappy-stubs-internal.h +116 -38
- data/vendor/snappy/snappy-stubs-public.h.in +20 -46
- data/vendor/snappy/snappy-test.cc +26 -22
- data/vendor/snappy/snappy-test.h +24 -98
- data/vendor/snappy/snappy.cc +380 -183
- data/vendor/snappy/snappy.h +14 -10
- data/vendor/snappy/snappy_compress_fuzzer.cc +59 -0
- data/vendor/snappy/snappy_uncompress_fuzzer.cc +57 -0
- data/vendor/snappy/snappy_unittest.cc +236 -261
- metadata +37 -92
- data/.travis.yml +0 -26
- data/smoke.sh +0 -8
- data/test/test-snappy-reader.rb +0 -129
- data/test/test-snappy-writer.rb +0 -55
- data/test/test-snappy.rb +0 -58
- data/vendor/snappy/ChangeLog +0 -2468
- data/vendor/snappy/INSTALL +0 -370
- data/vendor/snappy/Makefile +0 -982
- data/vendor/snappy/Makefile.am +0 -26
- data/vendor/snappy/Makefile.in +0 -982
- data/vendor/snappy/aclocal.m4 +0 -9738
- data/vendor/snappy/autogen.sh +0 -12
- data/vendor/snappy/autom4te.cache/output.0 +0 -18856
- data/vendor/snappy/autom4te.cache/output.1 +0 -18852
- data/vendor/snappy/autom4te.cache/requests +0 -297
- data/vendor/snappy/autom4te.cache/traces.0 +0 -2689
- data/vendor/snappy/autom4te.cache/traces.1 +0 -714
- data/vendor/snappy/config.guess +0 -1530
- data/vendor/snappy/config.h +0 -135
- data/vendor/snappy/config.h.in +0 -134
- data/vendor/snappy/config.log +0 -1640
- data/vendor/snappy/config.status +0 -2318
- data/vendor/snappy/config.sub +0 -1773
- data/vendor/snappy/configure +0 -18852
- data/vendor/snappy/configure.ac +0 -134
- data/vendor/snappy/depcomp +0 -688
- data/vendor/snappy/install-sh +0 -527
- data/vendor/snappy/libtool +0 -10246
- data/vendor/snappy/ltmain.sh +0 -9661
- data/vendor/snappy/m4/gtest.m4 +0 -74
- data/vendor/snappy/m4/libtool.m4 +0 -8001
- data/vendor/snappy/m4/ltoptions.m4 +0 -384
- data/vendor/snappy/m4/ltsugar.m4 +0 -123
- data/vendor/snappy/m4/ltversion.m4 +0 -23
- data/vendor/snappy/m4/lt~obsolete.m4 +0 -98
- data/vendor/snappy/missing +0 -331
- data/vendor/snappy/snappy-stubs-public.h +0 -100
- data/vendor/snappy/snappy.pc +0 -10
- data/vendor/snappy/snappy.pc.in +0 -10
- data/vendor/snappy/stamp-h1 +0 -1
@@ -36,21 +36,30 @@
|
|
36
36
|
namespace snappy {
|
37
37
|
namespace internal {
|
38
38
|
|
39
|
+
// Working memory performs a single allocation to hold all scratch space
|
40
|
+
// required for compression.
|
39
41
|
class WorkingMemory {
|
40
42
|
public:
|
41
|
-
WorkingMemory(
|
42
|
-
~WorkingMemory()
|
43
|
+
explicit WorkingMemory(size_t input_size);
|
44
|
+
~WorkingMemory();
|
43
45
|
|
44
46
|
// Allocates and clears a hash table using memory in "*this",
|
45
47
|
// stores the number of buckets in "*table_size" and returns a pointer to
|
46
48
|
// the base of the hash table.
|
47
|
-
uint16* GetHashTable(size_t
|
49
|
+
uint16* GetHashTable(size_t fragment_size, int* table_size) const;
|
50
|
+
char* GetScratchInput() const { return input_; }
|
51
|
+
char* GetScratchOutput() const { return output_; }
|
48
52
|
|
49
53
|
private:
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
+
char* mem_; // the allocated memory, never nullptr
|
55
|
+
size_t size_; // the size of the allocated memory, never 0
|
56
|
+
uint16* table_; // the pointer to the hashtable
|
57
|
+
char* input_; // the pointer to the input scratch buffer
|
58
|
+
char* output_; // the pointer to the output scratch buffer
|
59
|
+
|
60
|
+
// No copying
|
61
|
+
WorkingMemory(const WorkingMemory&);
|
62
|
+
void operator=(const WorkingMemory&);
|
54
63
|
};
|
55
64
|
|
56
65
|
// Flat array compression that does not emit the "uncompressed length"
|
@@ -80,9 +89,9 @@ char* CompressFragment(const char* input,
|
|
80
89
|
// Does not read *(s1 + (s2_limit - s2)) or beyond.
|
81
90
|
// Requires that s2_limit >= s2.
|
82
91
|
//
|
83
|
-
// Separate implementation for
|
84
|
-
|
85
|
-
|
92
|
+
// Separate implementation for 64-bit, little-endian cpus.
|
93
|
+
#if !defined(SNAPPY_IS_BIG_ENDIAN) && \
|
94
|
+
(defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM))
|
86
95
|
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
87
96
|
const char* s2,
|
88
97
|
const char* s2_limit) {
|
@@ -94,7 +103,7 @@ static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
|
94
103
|
// uncommon code paths that determine, without extra effort, whether the match
|
95
104
|
// length is less than 8. In short, we are hoping to avoid a conditional
|
96
105
|
// branch, and perhaps get better code layout from the C++ compiler.
|
97
|
-
if (
|
106
|
+
if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
98
107
|
uint64 a1 = UNALIGNED_LOAD64(s1);
|
99
108
|
uint64 a2 = UNALIGNED_LOAD64(s2);
|
100
109
|
if (a1 != a2) {
|
@@ -110,7 +119,7 @@ static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
|
110
119
|
// time until we find a 64-bit block that doesn't match; then we find
|
111
120
|
// the first non-matching bit and use that to calculate the total
|
112
121
|
// length of the match.
|
113
|
-
while (
|
122
|
+
while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
114
123
|
if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) {
|
115
124
|
s2 += 8;
|
116
125
|
matched += 8;
|
@@ -122,7 +131,7 @@ static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
|
122
131
|
return std::pair<size_t, bool>(matched, false);
|
123
132
|
}
|
124
133
|
}
|
125
|
-
while (
|
134
|
+
while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) {
|
126
135
|
if (s1[matched] == *s2) {
|
127
136
|
++s2;
|
128
137
|
++matched;
|
@@ -170,11 +179,6 @@ enum {
|
|
170
179
|
};
|
171
180
|
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
|
172
181
|
|
173
|
-
// Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits
|
174
|
-
static const uint32 wordmask[] = {
|
175
|
-
0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu
|
176
|
-
};
|
177
|
-
|
178
182
|
// Data stored per entry in lookup table:
|
179
183
|
// Range Bits-used Description
|
180
184
|
// ------------------------------------
|
@@ -45,6 +45,26 @@
|
|
45
45
|
#include <sys/mman.h>
|
46
46
|
#endif
|
47
47
|
|
48
|
+
#ifdef HAVE_UNISTD_H
|
49
|
+
#include <unistd.h>
|
50
|
+
#endif
|
51
|
+
|
52
|
+
#if defined(_MSC_VER)
|
53
|
+
#include <intrin.h>
|
54
|
+
#endif // defined(_MSC_VER)
|
55
|
+
|
56
|
+
#ifndef __has_feature
|
57
|
+
#define __has_feature(x) 0
|
58
|
+
#endif
|
59
|
+
|
60
|
+
#if __has_feature(memory_sanitizer)
|
61
|
+
#include <sanitizer/msan_interface.h>
|
62
|
+
#define SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \
|
63
|
+
__msan_unpoison((address), (size))
|
64
|
+
#else
|
65
|
+
#define SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) /* empty */
|
66
|
+
#endif // __has_feature(memory_sanitizer)
|
67
|
+
|
48
68
|
#include "snappy-stubs-public.h"
|
49
69
|
|
50
70
|
#if defined(__x86_64__)
|
@@ -52,6 +72,14 @@
|
|
52
72
|
// Enable 64-bit optimized versions of some routines.
|
53
73
|
#define ARCH_K8 1
|
54
74
|
|
75
|
+
#elif defined(__ppc64__)
|
76
|
+
|
77
|
+
#define ARCH_PPC 1
|
78
|
+
|
79
|
+
#elif defined(__aarch64__)
|
80
|
+
|
81
|
+
#define ARCH_ARM 1
|
82
|
+
|
55
83
|
#endif
|
56
84
|
|
57
85
|
// Needed by OS X, among others.
|
@@ -59,10 +87,6 @@
|
|
59
87
|
#define MAP_ANONYMOUS MAP_ANON
|
60
88
|
#endif
|
61
89
|
|
62
|
-
// Pull in std::min, std::ostream, and the likes. This is safe because this
|
63
|
-
// header file is never used from any public header files.
|
64
|
-
using namespace std;
|
65
|
-
|
66
90
|
// The size of an array, if known at compile-time.
|
67
91
|
// Will give unexpected results if used on a pointer.
|
68
92
|
// We undefine it first, since some compilers already have a definition.
|
@@ -73,11 +97,11 @@ using namespace std;
|
|
73
97
|
|
74
98
|
// Static prediction hints.
|
75
99
|
#ifdef HAVE_BUILTIN_EXPECT
|
76
|
-
#define
|
77
|
-
#define
|
100
|
+
#define SNAPPY_PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
101
|
+
#define SNAPPY_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
78
102
|
#else
|
79
|
-
#define
|
80
|
-
#define
|
103
|
+
#define SNAPPY_PREDICT_FALSE(x) x
|
104
|
+
#define SNAPPY_PREDICT_TRUE(x) x
|
81
105
|
#endif
|
82
106
|
|
83
107
|
// This is only used for recomputing the tag byte table used during
|
@@ -96,9 +120,10 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
|
|
96
120
|
|
97
121
|
// Potentially unaligned loads and stores.
|
98
122
|
|
99
|
-
// x86 and
|
123
|
+
// x86, PowerPC, and ARM64 can simply do these loads and stores native.
|
100
124
|
|
101
|
-
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__)
|
125
|
+
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \
|
126
|
+
defined(__aarch64__)
|
102
127
|
|
103
128
|
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
|
104
129
|
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
|
@@ -174,7 +199,7 @@ struct Unaligned32Struct {
|
|
174
199
|
((reinterpret_cast< ::snappy::base::internal::Unaligned32Struct *>(_p))->value = \
|
175
200
|
(_val))
|
176
201
|
|
177
|
-
// TODO
|
202
|
+
// TODO: NEON supports unaligned 64-bit loads and stores.
|
178
203
|
// See if that would be more efficient on platforms supporting it,
|
179
204
|
// at least for copies.
|
180
205
|
|
@@ -225,22 +250,8 @@ inline void UNALIGNED_STORE64(void *p, uint64 v) {
|
|
225
250
|
|
226
251
|
#endif
|
227
252
|
|
228
|
-
// This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64
|
229
|
-
// on some platforms, in particular ARM.
|
230
|
-
inline void UnalignedCopy64(const void *src, void *dst) {
|
231
|
-
if (sizeof(void *) == 8) {
|
232
|
-
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
|
233
|
-
} else {
|
234
|
-
const char *src_char = reinterpret_cast<const char *>(src);
|
235
|
-
char *dst_char = reinterpret_cast<char *>(dst);
|
236
|
-
|
237
|
-
UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char));
|
238
|
-
UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4));
|
239
|
-
}
|
240
|
-
}
|
241
|
-
|
242
253
|
// The following guarantees declaration of the byte swap functions.
|
243
|
-
#
|
254
|
+
#if defined(SNAPPY_IS_BIG_ENDIAN)
|
244
255
|
|
245
256
|
#ifdef HAVE_SYS_BYTEORDER_H
|
246
257
|
#include <sys/byteorder.h>
|
@@ -297,7 +308,7 @@ inline uint64 bswap_64(uint64 x) {
|
|
297
308
|
|
298
309
|
#endif
|
299
310
|
|
300
|
-
#endif //
|
311
|
+
#endif // defined(SNAPPY_IS_BIG_ENDIAN)
|
301
312
|
|
302
313
|
// Convert to little-endian storage, opposite of network format.
|
303
314
|
// Convert x from host to little endian: x = LittleEndian.FromHost(x);
|
@@ -311,7 +322,7 @@ inline uint64 bswap_64(uint64 x) {
|
|
311
322
|
class LittleEndian {
|
312
323
|
public:
|
313
324
|
// Conversion functions.
|
314
|
-
#
|
325
|
+
#if defined(SNAPPY_IS_BIG_ENDIAN)
|
315
326
|
|
316
327
|
static uint16 FromHost16(uint16 x) { return bswap_16(x); }
|
317
328
|
static uint16 ToHost16(uint16 x) { return bswap_16(x); }
|
@@ -321,7 +332,7 @@ class LittleEndian {
|
|
321
332
|
|
322
333
|
static bool IsLittleEndian() { return false; }
|
323
334
|
|
324
|
-
#else // !defined(
|
335
|
+
#else // !defined(SNAPPY_IS_BIG_ENDIAN)
|
325
336
|
|
326
337
|
static uint16 FromHost16(uint16 x) { return x; }
|
327
338
|
static uint16 ToHost16(uint16 x) { return x; }
|
@@ -331,7 +342,7 @@ class LittleEndian {
|
|
331
342
|
|
332
343
|
static bool IsLittleEndian() { return true; }
|
333
344
|
|
334
|
-
#endif // !defined(
|
345
|
+
#endif // !defined(SNAPPY_IS_BIG_ENDIAN)
|
335
346
|
|
336
347
|
// Functions to do unaligned loads and stores in little-endian order.
|
337
348
|
static uint16 Load16(const void *p) {
|
@@ -354,6 +365,9 @@ class LittleEndian {
|
|
354
365
|
// Some bit-manipulation functions.
|
355
366
|
class Bits {
|
356
367
|
public:
|
368
|
+
// Return floor(log2(n)) for positive integer n.
|
369
|
+
static int Log2FloorNonZero(uint32 n);
|
370
|
+
|
357
371
|
// Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0.
|
358
372
|
static int Log2Floor(uint32 n);
|
359
373
|
|
@@ -361,31 +375,85 @@ class Bits {
|
|
361
375
|
// undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except
|
362
376
|
// that it's 0-indexed.
|
363
377
|
static int FindLSBSetNonZero(uint32 n);
|
378
|
+
|
379
|
+
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
364
380
|
static int FindLSBSetNonZero64(uint64 n);
|
381
|
+
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
365
382
|
|
366
383
|
private:
|
367
|
-
|
384
|
+
// No copying
|
385
|
+
Bits(const Bits&);
|
386
|
+
void operator=(const Bits&);
|
368
387
|
};
|
369
388
|
|
370
389
|
#ifdef HAVE_BUILTIN_CTZ
|
371
390
|
|
391
|
+
inline int Bits::Log2FloorNonZero(uint32 n) {
|
392
|
+
assert(n != 0);
|
393
|
+
// (31 ^ x) is equivalent to (31 - x) for x in [0, 31]. An easy proof
|
394
|
+
// represents subtraction in base 2 and observes that there's no carry.
|
395
|
+
//
|
396
|
+
// GCC and Clang represent __builtin_clz on x86 as 31 ^ _bit_scan_reverse(x).
|
397
|
+
// Using "31 ^" here instead of "31 -" allows the optimizer to strip the
|
398
|
+
// function body down to _bit_scan_reverse(x).
|
399
|
+
return 31 ^ __builtin_clz(n);
|
400
|
+
}
|
401
|
+
|
372
402
|
inline int Bits::Log2Floor(uint32 n) {
|
373
|
-
return n == 0 ? -1 :
|
403
|
+
return (n == 0) ? -1 : Bits::Log2FloorNonZero(n);
|
374
404
|
}
|
375
405
|
|
376
406
|
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
407
|
+
assert(n != 0);
|
377
408
|
return __builtin_ctz(n);
|
378
409
|
}
|
379
410
|
|
411
|
+
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
380
412
|
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
413
|
+
assert(n != 0);
|
381
414
|
return __builtin_ctzll(n);
|
382
415
|
}
|
416
|
+
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
383
417
|
|
384
|
-
#
|
418
|
+
#elif defined(_MSC_VER)
|
419
|
+
|
420
|
+
inline int Bits::Log2FloorNonZero(uint32 n) {
|
421
|
+
assert(n != 0);
|
422
|
+
unsigned long where;
|
423
|
+
_BitScanReverse(&where, n);
|
424
|
+
return static_cast<int>(where);
|
425
|
+
}
|
385
426
|
|
386
427
|
inline int Bits::Log2Floor(uint32 n) {
|
387
|
-
|
388
|
-
|
428
|
+
unsigned long where;
|
429
|
+
if (_BitScanReverse(&where, n))
|
430
|
+
return static_cast<int>(where);
|
431
|
+
return -1;
|
432
|
+
}
|
433
|
+
|
434
|
+
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
435
|
+
assert(n != 0);
|
436
|
+
unsigned long where;
|
437
|
+
if (_BitScanForward(&where, n))
|
438
|
+
return static_cast<int>(where);
|
439
|
+
return 32;
|
440
|
+
}
|
441
|
+
|
442
|
+
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
443
|
+
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
444
|
+
assert(n != 0);
|
445
|
+
unsigned long where;
|
446
|
+
if (_BitScanForward64(&where, n))
|
447
|
+
return static_cast<int>(where);
|
448
|
+
return 64;
|
449
|
+
}
|
450
|
+
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
451
|
+
|
452
|
+
#else // Portable versions.
|
453
|
+
|
454
|
+
inline int Bits::Log2FloorNonZero(uint32 n) {
|
455
|
+
assert(n != 0);
|
456
|
+
|
389
457
|
int log = 0;
|
390
458
|
uint32 value = n;
|
391
459
|
for (int i = 4; i >= 0; --i) {
|
@@ -400,7 +468,13 @@ inline int Bits::Log2Floor(uint32 n) {
|
|
400
468
|
return log;
|
401
469
|
}
|
402
470
|
|
471
|
+
inline int Bits::Log2Floor(uint32 n) {
|
472
|
+
return (n == 0) ? -1 : Bits::Log2FloorNonZero(n);
|
473
|
+
}
|
474
|
+
|
403
475
|
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
476
|
+
assert(n != 0);
|
477
|
+
|
404
478
|
int rc = 31;
|
405
479
|
for (int i = 4, shift = 1 << 4; i >= 0; --i) {
|
406
480
|
const uint32 x = n << shift;
|
@@ -413,8 +487,11 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
|
|
413
487
|
return rc;
|
414
488
|
}
|
415
489
|
|
490
|
+
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
416
491
|
// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero().
|
417
492
|
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
493
|
+
assert(n != 0);
|
494
|
+
|
418
495
|
const uint32 bottombits = static_cast<uint32>(n);
|
419
496
|
if (bottombits == 0) {
|
420
497
|
// Bottom bits are zero, so scan in top bits
|
@@ -423,6 +500,7 @@ inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
|
423
500
|
return FindLSBSetNonZero(bottombits);
|
424
501
|
}
|
425
502
|
}
|
503
|
+
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
426
504
|
|
427
505
|
#endif // End portable versions.
|
428
506
|
|
@@ -446,7 +524,7 @@ class Varint {
|
|
446
524
|
static char* Encode32(char* ptr, uint32 v);
|
447
525
|
|
448
526
|
// EFFECTS Appends the varint representation of "value" to "*s".
|
449
|
-
static void Append32(string* s, uint32 value);
|
527
|
+
static void Append32(std::string* s, uint32 value);
|
450
528
|
};
|
451
529
|
|
452
530
|
inline const char* Varint::Parse32WithLimit(const char* p,
|
@@ -503,7 +581,7 @@ inline char* Varint::Encode32(char* sptr, uint32 v) {
|
|
503
581
|
// replace this function with one that resizes the string without
|
504
582
|
// filling the new space with zeros (if applicable) --
|
505
583
|
// it will be non-portable but faster.
|
506
|
-
inline void STLStringResizeUninitialized(string* s, size_t new_size) {
|
584
|
+
inline void STLStringResizeUninitialized(std::string* s, size_t new_size) {
|
507
585
|
s->resize(new_size);
|
508
586
|
}
|
509
587
|
|
@@ -519,7 +597,7 @@ inline void STLStringResizeUninitialized(string* s, size_t new_size) {
|
|
519
597
|
// (http://www.open-std.org/JTC1/SC22/WG21/docs/lwg-defects.html#530)
|
520
598
|
// proposes this as the method. It will officially be part of the standard
|
521
599
|
// for C++0x. This should already work on all current implementations.
|
522
|
-
inline char* string_as_array(string* str) {
|
600
|
+
inline char* string_as_array(std::string* str) {
|
523
601
|
return str->empty() ? NULL : &*str->begin();
|
524
602
|
}
|
525
603
|
|
@@ -1,5 +1,4 @@
|
|
1
1
|
// Copyright 2011 Google Inc. All Rights Reserved.
|
2
|
-
// Author: sesse@google.com (Steinar H. Gunderson)
|
3
2
|
//
|
4
3
|
// Redistribution and use in source and binary forms, with or without
|
5
4
|
// modification, are permitted provided that the following conditions are
|
@@ -36,64 +35,39 @@
|
|
36
35
|
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
37
36
|
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
38
37
|
|
39
|
-
#
|
40
|
-
#include <
|
41
|
-
#
|
42
|
-
|
43
|
-
#if @ac_cv_have_stddef_h@
|
44
|
-
#include <stddef.h>
|
45
|
-
#endif
|
38
|
+
#include <cstddef>
|
39
|
+
#include <cstdint>
|
40
|
+
#include <string>
|
46
41
|
|
47
|
-
#if
|
42
|
+
#if ${HAVE_SYS_UIO_H_01} // HAVE_SYS_UIO_H
|
48
43
|
#include <sys/uio.h>
|
49
|
-
#endif
|
44
|
+
#endif // HAVE_SYS_UIO_H
|
50
45
|
|
51
|
-
#define SNAPPY_MAJOR
|
52
|
-
#define SNAPPY_MINOR
|
53
|
-
#define SNAPPY_PATCHLEVEL
|
46
|
+
#define SNAPPY_MAJOR ${PROJECT_VERSION_MAJOR}
|
47
|
+
#define SNAPPY_MINOR ${PROJECT_VERSION_MINOR}
|
48
|
+
#define SNAPPY_PATCHLEVEL ${PROJECT_VERSION_PATCH}
|
54
49
|
#define SNAPPY_VERSION \
|
55
50
|
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
|
56
51
|
|
57
|
-
#include <string>
|
58
|
-
|
59
52
|
namespace snappy {
|
60
53
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
typedef uint64_t uint64;
|
70
|
-
#else
|
71
|
-
typedef signed char int8;
|
72
|
-
typedef unsigned char uint8;
|
73
|
-
typedef short int16;
|
74
|
-
typedef unsigned short uint16;
|
75
|
-
typedef int int32;
|
76
|
-
typedef unsigned int uint32;
|
77
|
-
typedef long long int64;
|
78
|
-
typedef unsigned long long uint64;
|
79
|
-
#endif
|
80
|
-
|
81
|
-
typedef std::string string;
|
82
|
-
|
83
|
-
#ifndef DISALLOW_COPY_AND_ASSIGN
|
84
|
-
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
85
|
-
TypeName(const TypeName&); \
|
86
|
-
void operator=(const TypeName&)
|
87
|
-
#endif
|
54
|
+
using int8 = std::int8_t;
|
55
|
+
using uint8 = std::uint8_t;
|
56
|
+
using int16 = std::int16_t;
|
57
|
+
using uint16 = std::uint16_t;
|
58
|
+
using int32 = std::int32_t;
|
59
|
+
using uint32 = std::uint32_t;
|
60
|
+
using int64 = std::int64_t;
|
61
|
+
using uint64 = std::uint64_t;
|
88
62
|
|
89
|
-
#if
|
63
|
+
#if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H
|
90
64
|
// Windows does not have an iovec type, yet the concept is universally useful.
|
91
65
|
// It is simple to define it ourselves, so we put it inside our own namespace.
|
92
66
|
struct iovec {
|
93
|
-
|
94
|
-
|
67
|
+
void* iov_base;
|
68
|
+
size_t iov_len;
|
95
69
|
};
|
96
|
-
#endif
|
70
|
+
#endif // !HAVE_SYS_UIO_H
|
97
71
|
|
98
72
|
} // namespace snappy
|
99
73
|
|
@@ -33,6 +33,9 @@
|
|
33
33
|
#endif
|
34
34
|
|
35
35
|
#ifdef HAVE_WINDOWS_H
|
36
|
+
// Needed to be able to use std::max without workarounds in the source code.
|
37
|
+
// https://support.microsoft.com/en-us/help/143208/prb-using-stl-in-windows-program-can-cause-min-max-conflicts
|
38
|
+
#define NOMINMAX
|
36
39
|
#include <windows.h>
|
37
40
|
#endif
|
38
41
|
|
@@ -45,12 +48,12 @@ DEFINE_bool(run_microbenchmarks, true,
|
|
45
48
|
|
46
49
|
namespace snappy {
|
47
50
|
|
48
|
-
string ReadTestDataFile(const string& base, size_t size_limit) {
|
49
|
-
string contents;
|
51
|
+
std::string ReadTestDataFile(const std::string& base, size_t size_limit) {
|
52
|
+
std::string contents;
|
50
53
|
const char* srcdir = getenv("srcdir"); // This is set by Automake.
|
51
|
-
string prefix;
|
54
|
+
std::string prefix;
|
52
55
|
if (srcdir) {
|
53
|
-
prefix = string(srcdir) + "/";
|
56
|
+
prefix = std::string(srcdir) + "/";
|
54
57
|
}
|
55
58
|
file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
|
56
59
|
).CheckSuccess();
|
@@ -60,11 +63,11 @@ string ReadTestDataFile(const string& base, size_t size_limit) {
|
|
60
63
|
return contents;
|
61
64
|
}
|
62
65
|
|
63
|
-
string ReadTestDataFile(const string& base) {
|
66
|
+
std::string ReadTestDataFile(const std::string& base) {
|
64
67
|
return ReadTestDataFile(base, 0);
|
65
68
|
}
|
66
69
|
|
67
|
-
string
|
70
|
+
std::string StrFormat(const char* format, ...) {
|
68
71
|
char buf[4096];
|
69
72
|
va_list ap;
|
70
73
|
va_start(ap, format);
|
@@ -76,7 +79,7 @@ string StringPrintf(const char* format, ...) {
|
|
76
79
|
bool benchmark_running = false;
|
77
80
|
int64 benchmark_real_time_us = 0;
|
78
81
|
int64 benchmark_cpu_time_us = 0;
|
79
|
-
string
|
82
|
+
std::string* benchmark_label = nullptr;
|
80
83
|
int64 benchmark_bytes_processed = 0;
|
81
84
|
|
82
85
|
void ResetBenchmarkTiming() {
|
@@ -160,11 +163,11 @@ void StopBenchmarkTiming() {
|
|
160
163
|
benchmark_running = false;
|
161
164
|
}
|
162
165
|
|
163
|
-
void SetBenchmarkLabel(const string& str) {
|
166
|
+
void SetBenchmarkLabel(const std::string& str) {
|
164
167
|
if (benchmark_label) {
|
165
168
|
delete benchmark_label;
|
166
169
|
}
|
167
|
-
benchmark_label = new string(str);
|
170
|
+
benchmark_label = new std::string(str);
|
168
171
|
}
|
169
172
|
|
170
173
|
void SetBenchmarkBytesProcessed(int64 bytes) {
|
@@ -201,7 +204,7 @@ void Benchmark::Run() {
|
|
201
204
|
if (benchmark_real_time_us > 0) {
|
202
205
|
num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
|
203
206
|
}
|
204
|
-
num_iterations = max(num_iterations, kCalibrateIterations);
|
207
|
+
num_iterations = std::max(num_iterations, kCalibrateIterations);
|
205
208
|
BenchmarkRun benchmark_runs[kNumRuns];
|
206
209
|
|
207
210
|
for (int run = 0; run < kNumRuns; ++run) {
|
@@ -214,13 +217,13 @@ void Benchmark::Run() {
|
|
214
217
|
benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us;
|
215
218
|
}
|
216
219
|
|
217
|
-
string heading =
|
218
|
-
string human_readable_speed;
|
220
|
+
std::string heading = StrFormat("%s/%d", name_.c_str(), test_case_num);
|
221
|
+
std::string human_readable_speed;
|
219
222
|
|
220
|
-
nth_element(benchmark_runs,
|
221
|
-
|
222
|
-
|
223
|
-
|
223
|
+
std::nth_element(benchmark_runs,
|
224
|
+
benchmark_runs + kMedianPos,
|
225
|
+
benchmark_runs + kNumRuns,
|
226
|
+
BenchmarkCompareCPUTime());
|
224
227
|
int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
|
225
228
|
int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
|
226
229
|
if (cpu_time_us <= 0) {
|
@@ -229,15 +232,16 @@ void Benchmark::Run() {
|
|
229
232
|
int64 bytes_per_second =
|
230
233
|
benchmark_bytes_processed * 1000000 / cpu_time_us;
|
231
234
|
if (bytes_per_second < 1024) {
|
232
|
-
human_readable_speed =
|
235
|
+
human_readable_speed =
|
236
|
+
StrFormat("%dB/s", static_cast<int>(bytes_per_second));
|
233
237
|
} else if (bytes_per_second < 1024 * 1024) {
|
234
|
-
human_readable_speed =
|
238
|
+
human_readable_speed = StrFormat(
|
235
239
|
"%.1fkB/s", bytes_per_second / 1024.0f);
|
236
240
|
} else if (bytes_per_second < 1024 * 1024 * 1024) {
|
237
|
-
human_readable_speed =
|
241
|
+
human_readable_speed = StrFormat(
|
238
242
|
"%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
|
239
243
|
} else {
|
240
|
-
human_readable_speed =
|
244
|
+
human_readable_speed = StrFormat(
|
241
245
|
"%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
|
242
246
|
}
|
243
247
|
}
|
@@ -523,8 +527,8 @@ int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
|
|
523
527
|
LOG(WARNING)
|
524
528
|
<< "UncompressChunkOrAll: Received some extra data, bytes total: "
|
525
529
|
<< uncomp_stream_.avail_in << " bytes: "
|
526
|
-
<< string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
527
|
-
|
530
|
+
<< std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
531
|
+
std::min(int(uncomp_stream_.avail_in), 20));
|
528
532
|
UncompressErrorInit();
|
529
533
|
return Z_DATA_ERROR; // what's the extra data for?
|
530
534
|
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
|