snappy 0.0.12 → 0.0.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/snappy/version.rb +1 -1
- data/lib/snappy/writer.rb +6 -0
- metadata +3 -41
- data/vendor/snappy/AUTHORS +0 -1
- data/vendor/snappy/COPYING +0 -54
- data/vendor/snappy/ChangeLog +0 -1916
- data/vendor/snappy/Makefile.am +0 -23
- data/vendor/snappy/NEWS +0 -128
- data/vendor/snappy/README +0 -135
- data/vendor/snappy/autogen.sh +0 -7
- data/vendor/snappy/configure.ac +0 -133
- data/vendor/snappy/format_description.txt +0 -110
- data/vendor/snappy/framing_format.txt +0 -135
- data/vendor/snappy/m4/gtest.m4 +0 -74
- data/vendor/snappy/snappy-c.cc +0 -90
- data/vendor/snappy/snappy-c.h +0 -138
- data/vendor/snappy/snappy-internal.h +0 -150
- data/vendor/snappy/snappy-sinksource.cc +0 -71
- data/vendor/snappy/snappy-sinksource.h +0 -137
- data/vendor/snappy/snappy-stubs-internal.cc +0 -42
- data/vendor/snappy/snappy-stubs-internal.h +0 -491
- data/vendor/snappy/snappy-stubs-public.h.in +0 -98
- data/vendor/snappy/snappy-test.cc +0 -606
- data/vendor/snappy/snappy-test.h +0 -582
- data/vendor/snappy/snappy.cc +0 -1306
- data/vendor/snappy/snappy.h +0 -184
- data/vendor/snappy/snappy_unittest.cc +0 -1355
- data/vendor/snappy/testdata/alice29.txt +0 -3609
- data/vendor/snappy/testdata/asyoulik.txt +0 -4122
- data/vendor/snappy/testdata/baddata1.snappy +0 -0
- data/vendor/snappy/testdata/baddata2.snappy +0 -0
- data/vendor/snappy/testdata/baddata3.snappy +0 -0
- data/vendor/snappy/testdata/fireworks.jpeg +0 -0
- data/vendor/snappy/testdata/geo.protodata +0 -0
- data/vendor/snappy/testdata/html +0 -1
- data/vendor/snappy/testdata/html_x_4 +0 -1
- data/vendor/snappy/testdata/kppkn.gtb +0 -0
- data/vendor/snappy/testdata/lcet10.txt +0 -7519
- data/vendor/snappy/testdata/paper-100k.pdf +2 -600
- data/vendor/snappy/testdata/plrabn12.txt +0 -10699
- data/vendor/snappy/testdata/urls.10K +0 -10000
@@ -1,98 +0,0 @@
|
|
1
|
-
// Copyright 2011 Google Inc. All Rights Reserved.
|
2
|
-
// Author: sesse@google.com (Steinar H. Gunderson)
|
3
|
-
//
|
4
|
-
// Redistribution and use in source and binary forms, with or without
|
5
|
-
// modification, are permitted provided that the following conditions are
|
6
|
-
// met:
|
7
|
-
//
|
8
|
-
// * Redistributions of source code must retain the above copyright
|
9
|
-
// notice, this list of conditions and the following disclaimer.
|
10
|
-
// * Redistributions in binary form must reproduce the above
|
11
|
-
// copyright notice, this list of conditions and the following disclaimer
|
12
|
-
// in the documentation and/or other materials provided with the
|
13
|
-
// distribution.
|
14
|
-
// * Neither the name of Google Inc. nor the names of its
|
15
|
-
// contributors may be used to endorse or promote products derived from
|
16
|
-
// this software without specific prior written permission.
|
17
|
-
//
|
18
|
-
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
-
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
-
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
-
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
-
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
-
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
-
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
-
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
-
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
-
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
-
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
-
//
|
30
|
-
// Various type stubs for the open-source version of Snappy.
|
31
|
-
//
|
32
|
-
// This file cannot include config.h, as it is included from snappy.h,
|
33
|
-
// which is a public header. Instead, snappy-stubs-public.h is generated by
|
34
|
-
// from snappy-stubs-public.h.in at configure time.
|
35
|
-
|
36
|
-
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
37
|
-
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
38
|
-
|
39
|
-
#if @ac_cv_have_stdint_h@
|
40
|
-
#include <stdint.h>
|
41
|
-
#endif
|
42
|
-
|
43
|
-
#if @ac_cv_have_stddef_h@
|
44
|
-
#include <stddef.h>
|
45
|
-
#endif
|
46
|
-
|
47
|
-
#if @ac_cv_have_sys_uio_h@
|
48
|
-
#include <sys/uio.h>
|
49
|
-
#endif
|
50
|
-
|
51
|
-
#define SNAPPY_MAJOR @SNAPPY_MAJOR@
|
52
|
-
#define SNAPPY_MINOR @SNAPPY_MINOR@
|
53
|
-
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
|
54
|
-
#define SNAPPY_VERSION \
|
55
|
-
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
|
56
|
-
|
57
|
-
#include <string>
|
58
|
-
|
59
|
-
namespace snappy {
|
60
|
-
|
61
|
-
#if @ac_cv_have_stdint_h@
|
62
|
-
typedef int8_t int8;
|
63
|
-
typedef uint8_t uint8;
|
64
|
-
typedef int16_t int16;
|
65
|
-
typedef uint16_t uint16;
|
66
|
-
typedef int32_t int32;
|
67
|
-
typedef uint32_t uint32;
|
68
|
-
typedef int64_t int64;
|
69
|
-
typedef uint64_t uint64;
|
70
|
-
#else
|
71
|
-
typedef signed char int8;
|
72
|
-
typedef unsigned char uint8;
|
73
|
-
typedef short int16;
|
74
|
-
typedef unsigned short uint16;
|
75
|
-
typedef int int32;
|
76
|
-
typedef unsigned int uint32;
|
77
|
-
typedef long long int64;
|
78
|
-
typedef unsigned long long uint64;
|
79
|
-
#endif
|
80
|
-
|
81
|
-
typedef std::string string;
|
82
|
-
|
83
|
-
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
84
|
-
TypeName(const TypeName&); \
|
85
|
-
void operator=(const TypeName&)
|
86
|
-
|
87
|
-
#if !@ac_cv_have_sys_uio_h@
|
88
|
-
// Windows does not have an iovec type, yet the concept is universally useful.
|
89
|
-
// It is simple to define it ourselves, so we put it inside our own namespace.
|
90
|
-
struct iovec {
|
91
|
-
void* iov_base;
|
92
|
-
size_t iov_len;
|
93
|
-
};
|
94
|
-
#endif
|
95
|
-
|
96
|
-
} // namespace snappy
|
97
|
-
|
98
|
-
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
@@ -1,606 +0,0 @@
|
|
1
|
-
// Copyright 2011 Google Inc. All Rights Reserved.
|
2
|
-
//
|
3
|
-
// Redistribution and use in source and binary forms, with or without
|
4
|
-
// modification, are permitted provided that the following conditions are
|
5
|
-
// met:
|
6
|
-
//
|
7
|
-
// * Redistributions of source code must retain the above copyright
|
8
|
-
// notice, this list of conditions and the following disclaimer.
|
9
|
-
// * Redistributions in binary form must reproduce the above
|
10
|
-
// copyright notice, this list of conditions and the following disclaimer
|
11
|
-
// in the documentation and/or other materials provided with the
|
12
|
-
// distribution.
|
13
|
-
// * Neither the name of Google Inc. nor the names of its
|
14
|
-
// contributors may be used to endorse or promote products derived from
|
15
|
-
// this software without specific prior written permission.
|
16
|
-
//
|
17
|
-
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
18
|
-
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
19
|
-
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
20
|
-
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
21
|
-
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
22
|
-
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
23
|
-
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
24
|
-
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
25
|
-
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
26
|
-
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
27
|
-
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
28
|
-
//
|
29
|
-
// Various stubs for the unit tests for the open-source version of Snappy.
|
30
|
-
|
31
|
-
#include "snappy-test.h"
|
32
|
-
|
33
|
-
#ifdef HAVE_WINDOWS_H
|
34
|
-
#define WIN32_LEAN_AND_MEAN
|
35
|
-
#include <windows.h>
|
36
|
-
#endif
|
37
|
-
|
38
|
-
#include <algorithm>
|
39
|
-
|
40
|
-
DEFINE_bool(run_microbenchmarks, true,
|
41
|
-
"Run microbenchmarks before doing anything else.");
|
42
|
-
|
43
|
-
namespace snappy {
|
44
|
-
|
45
|
-
string ReadTestDataFile(const string& base, size_t size_limit) {
|
46
|
-
string contents;
|
47
|
-
const char* srcdir = getenv("srcdir"); // This is set by Automake.
|
48
|
-
string prefix;
|
49
|
-
if (srcdir) {
|
50
|
-
prefix = string(srcdir) + "/";
|
51
|
-
}
|
52
|
-
file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
|
53
|
-
).CheckSuccess();
|
54
|
-
if (size_limit > 0) {
|
55
|
-
contents = contents.substr(0, size_limit);
|
56
|
-
}
|
57
|
-
return contents;
|
58
|
-
}
|
59
|
-
|
60
|
-
string ReadTestDataFile(const string& base) {
|
61
|
-
return ReadTestDataFile(base, 0);
|
62
|
-
}
|
63
|
-
|
64
|
-
string StringPrintf(const char* format, ...) {
|
65
|
-
char buf[4096];
|
66
|
-
va_list ap;
|
67
|
-
va_start(ap, format);
|
68
|
-
vsnprintf(buf, sizeof(buf), format, ap);
|
69
|
-
va_end(ap);
|
70
|
-
return buf;
|
71
|
-
}
|
72
|
-
|
73
|
-
bool benchmark_running = false;
|
74
|
-
int64 benchmark_real_time_us = 0;
|
75
|
-
int64 benchmark_cpu_time_us = 0;
|
76
|
-
string *benchmark_label = NULL;
|
77
|
-
int64 benchmark_bytes_processed = 0;
|
78
|
-
|
79
|
-
void ResetBenchmarkTiming() {
|
80
|
-
benchmark_real_time_us = 0;
|
81
|
-
benchmark_cpu_time_us = 0;
|
82
|
-
}
|
83
|
-
|
84
|
-
#ifdef WIN32
|
85
|
-
LARGE_INTEGER benchmark_start_real;
|
86
|
-
FILETIME benchmark_start_cpu;
|
87
|
-
#else // WIN32
|
88
|
-
struct timeval benchmark_start_real;
|
89
|
-
struct rusage benchmark_start_cpu;
|
90
|
-
#endif // WIN32
|
91
|
-
|
92
|
-
void StartBenchmarkTiming() {
|
93
|
-
#ifdef WIN32
|
94
|
-
QueryPerformanceCounter(&benchmark_start_real);
|
95
|
-
FILETIME dummy;
|
96
|
-
CHECK(GetProcessTimes(
|
97
|
-
GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_start_cpu));
|
98
|
-
#else
|
99
|
-
gettimeofday(&benchmark_start_real, NULL);
|
100
|
-
if (getrusage(RUSAGE_SELF, &benchmark_start_cpu) == -1) {
|
101
|
-
perror("getrusage(RUSAGE_SELF)");
|
102
|
-
exit(1);
|
103
|
-
}
|
104
|
-
#endif
|
105
|
-
benchmark_running = true;
|
106
|
-
}
|
107
|
-
|
108
|
-
void StopBenchmarkTiming() {
|
109
|
-
if (!benchmark_running) {
|
110
|
-
return;
|
111
|
-
}
|
112
|
-
|
113
|
-
#ifdef WIN32
|
114
|
-
LARGE_INTEGER benchmark_stop_real;
|
115
|
-
LARGE_INTEGER benchmark_frequency;
|
116
|
-
QueryPerformanceCounter(&benchmark_stop_real);
|
117
|
-
QueryPerformanceFrequency(&benchmark_frequency);
|
118
|
-
|
119
|
-
double elapsed_real = static_cast<double>(
|
120
|
-
benchmark_stop_real.QuadPart - benchmark_start_real.QuadPart) /
|
121
|
-
benchmark_frequency.QuadPart;
|
122
|
-
benchmark_real_time_us += elapsed_real * 1e6 + 0.5;
|
123
|
-
|
124
|
-
FILETIME benchmark_stop_cpu, dummy;
|
125
|
-
CHECK(GetProcessTimes(
|
126
|
-
GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_stop_cpu));
|
127
|
-
|
128
|
-
ULARGE_INTEGER start_ulargeint;
|
129
|
-
start_ulargeint.LowPart = benchmark_start_cpu.dwLowDateTime;
|
130
|
-
start_ulargeint.HighPart = benchmark_start_cpu.dwHighDateTime;
|
131
|
-
|
132
|
-
ULARGE_INTEGER stop_ulargeint;
|
133
|
-
stop_ulargeint.LowPart = benchmark_stop_cpu.dwLowDateTime;
|
134
|
-
stop_ulargeint.HighPart = benchmark_stop_cpu.dwHighDateTime;
|
135
|
-
|
136
|
-
benchmark_cpu_time_us +=
|
137
|
-
(stop_ulargeint.QuadPart - start_ulargeint.QuadPart + 5) / 10;
|
138
|
-
#else // WIN32
|
139
|
-
struct timeval benchmark_stop_real;
|
140
|
-
gettimeofday(&benchmark_stop_real, NULL);
|
141
|
-
benchmark_real_time_us +=
|
142
|
-
1000000 * (benchmark_stop_real.tv_sec - benchmark_start_real.tv_sec);
|
143
|
-
benchmark_real_time_us +=
|
144
|
-
(benchmark_stop_real.tv_usec - benchmark_start_real.tv_usec);
|
145
|
-
|
146
|
-
struct rusage benchmark_stop_cpu;
|
147
|
-
if (getrusage(RUSAGE_SELF, &benchmark_stop_cpu) == -1) {
|
148
|
-
perror("getrusage(RUSAGE_SELF)");
|
149
|
-
exit(1);
|
150
|
-
}
|
151
|
-
benchmark_cpu_time_us += 1000000 * (benchmark_stop_cpu.ru_utime.tv_sec -
|
152
|
-
benchmark_start_cpu.ru_utime.tv_sec);
|
153
|
-
benchmark_cpu_time_us += (benchmark_stop_cpu.ru_utime.tv_usec -
|
154
|
-
benchmark_start_cpu.ru_utime.tv_usec);
|
155
|
-
#endif // WIN32
|
156
|
-
|
157
|
-
benchmark_running = false;
|
158
|
-
}
|
159
|
-
|
160
|
-
void SetBenchmarkLabel(const string& str) {
|
161
|
-
if (benchmark_label) {
|
162
|
-
delete benchmark_label;
|
163
|
-
}
|
164
|
-
benchmark_label = new string(str);
|
165
|
-
}
|
166
|
-
|
167
|
-
void SetBenchmarkBytesProcessed(int64 bytes) {
|
168
|
-
benchmark_bytes_processed = bytes;
|
169
|
-
}
|
170
|
-
|
171
|
-
struct BenchmarkRun {
|
172
|
-
int64 real_time_us;
|
173
|
-
int64 cpu_time_us;
|
174
|
-
};
|
175
|
-
|
176
|
-
struct BenchmarkCompareCPUTime {
|
177
|
-
bool operator() (const BenchmarkRun& a, const BenchmarkRun& b) const {
|
178
|
-
return a.cpu_time_us < b.cpu_time_us;
|
179
|
-
}
|
180
|
-
};
|
181
|
-
|
182
|
-
void Benchmark::Run() {
|
183
|
-
for (int test_case_num = start_; test_case_num <= stop_; ++test_case_num) {
|
184
|
-
// Run a few iterations first to find out approximately how fast
|
185
|
-
// the benchmark is.
|
186
|
-
const int kCalibrateIterations = 100;
|
187
|
-
ResetBenchmarkTiming();
|
188
|
-
StartBenchmarkTiming();
|
189
|
-
(*function_)(kCalibrateIterations, test_case_num);
|
190
|
-
StopBenchmarkTiming();
|
191
|
-
|
192
|
-
// Let each test case run for about 200ms, but at least as many
|
193
|
-
// as we used to calibrate.
|
194
|
-
// Run five times and pick the median.
|
195
|
-
const int kNumRuns = 5;
|
196
|
-
const int kMedianPos = kNumRuns / 2;
|
197
|
-
int num_iterations = 0;
|
198
|
-
if (benchmark_real_time_us > 0) {
|
199
|
-
num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
|
200
|
-
}
|
201
|
-
num_iterations = max(num_iterations, kCalibrateIterations);
|
202
|
-
BenchmarkRun benchmark_runs[kNumRuns];
|
203
|
-
|
204
|
-
for (int run = 0; run < kNumRuns; ++run) {
|
205
|
-
ResetBenchmarkTiming();
|
206
|
-
StartBenchmarkTiming();
|
207
|
-
(*function_)(num_iterations, test_case_num);
|
208
|
-
StopBenchmarkTiming();
|
209
|
-
|
210
|
-
benchmark_runs[run].real_time_us = benchmark_real_time_us;
|
211
|
-
benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us;
|
212
|
-
}
|
213
|
-
|
214
|
-
string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
|
215
|
-
string human_readable_speed;
|
216
|
-
|
217
|
-
nth_element(benchmark_runs,
|
218
|
-
benchmark_runs + kMedianPos,
|
219
|
-
benchmark_runs + kNumRuns,
|
220
|
-
BenchmarkCompareCPUTime());
|
221
|
-
int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
|
222
|
-
int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
|
223
|
-
if (cpu_time_us <= 0) {
|
224
|
-
human_readable_speed = "?";
|
225
|
-
} else {
|
226
|
-
int64 bytes_per_second =
|
227
|
-
benchmark_bytes_processed * 1000000 / cpu_time_us;
|
228
|
-
if (bytes_per_second < 1024) {
|
229
|
-
human_readable_speed = StringPrintf("%dB/s", bytes_per_second);
|
230
|
-
} else if (bytes_per_second < 1024 * 1024) {
|
231
|
-
human_readable_speed = StringPrintf(
|
232
|
-
"%.1fkB/s", bytes_per_second / 1024.0f);
|
233
|
-
} else if (bytes_per_second < 1024 * 1024 * 1024) {
|
234
|
-
human_readable_speed = StringPrintf(
|
235
|
-
"%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
|
236
|
-
} else {
|
237
|
-
human_readable_speed = StringPrintf(
|
238
|
-
"%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
|
239
|
-
}
|
240
|
-
}
|
241
|
-
|
242
|
-
fprintf(stderr,
|
243
|
-
#ifdef WIN32
|
244
|
-
"%-18s %10I64d %10I64d %10d %s %s\n",
|
245
|
-
#else
|
246
|
-
"%-18s %10lld %10lld %10d %s %s\n",
|
247
|
-
#endif
|
248
|
-
heading.c_str(),
|
249
|
-
static_cast<long long>(real_time_us * 1000 / num_iterations),
|
250
|
-
static_cast<long long>(cpu_time_us * 1000 / num_iterations),
|
251
|
-
num_iterations,
|
252
|
-
human_readable_speed.c_str(),
|
253
|
-
benchmark_label->c_str());
|
254
|
-
}
|
255
|
-
}
|
256
|
-
|
257
|
-
#ifdef HAVE_LIBZ
|
258
|
-
|
259
|
-
ZLib::ZLib()
|
260
|
-
: comp_init_(false),
|
261
|
-
uncomp_init_(false) {
|
262
|
-
Reinit();
|
263
|
-
}
|
264
|
-
|
265
|
-
ZLib::~ZLib() {
|
266
|
-
if (comp_init_) { deflateEnd(&comp_stream_); }
|
267
|
-
if (uncomp_init_) { inflateEnd(&uncomp_stream_); }
|
268
|
-
}
|
269
|
-
|
270
|
-
void ZLib::Reinit() {
|
271
|
-
compression_level_ = Z_DEFAULT_COMPRESSION;
|
272
|
-
window_bits_ = MAX_WBITS;
|
273
|
-
mem_level_ = 8; // DEF_MEM_LEVEL
|
274
|
-
if (comp_init_) {
|
275
|
-
deflateEnd(&comp_stream_);
|
276
|
-
comp_init_ = false;
|
277
|
-
}
|
278
|
-
if (uncomp_init_) {
|
279
|
-
inflateEnd(&uncomp_stream_);
|
280
|
-
uncomp_init_ = false;
|
281
|
-
}
|
282
|
-
first_chunk_ = true;
|
283
|
-
}
|
284
|
-
|
285
|
-
void ZLib::Reset() {
|
286
|
-
first_chunk_ = true;
|
287
|
-
}
|
288
|
-
|
289
|
-
// --------- COMPRESS MODE
|
290
|
-
|
291
|
-
// Initialization method to be called if we hit an error while
|
292
|
-
// compressing. On hitting an error, call this method before returning
|
293
|
-
// the error.
|
294
|
-
void ZLib::CompressErrorInit() {
|
295
|
-
deflateEnd(&comp_stream_);
|
296
|
-
comp_init_ = false;
|
297
|
-
Reset();
|
298
|
-
}
|
299
|
-
|
300
|
-
int ZLib::DeflateInit() {
|
301
|
-
return deflateInit2(&comp_stream_,
|
302
|
-
compression_level_,
|
303
|
-
Z_DEFLATED,
|
304
|
-
window_bits_,
|
305
|
-
mem_level_,
|
306
|
-
Z_DEFAULT_STRATEGY);
|
307
|
-
}
|
308
|
-
|
309
|
-
int ZLib::CompressInit(Bytef *dest, uLongf *destLen,
|
310
|
-
const Bytef *source, uLong *sourceLen) {
|
311
|
-
int err;
|
312
|
-
|
313
|
-
comp_stream_.next_in = (Bytef*)source;
|
314
|
-
comp_stream_.avail_in = (uInt)*sourceLen;
|
315
|
-
if ((uLong)comp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
|
316
|
-
comp_stream_.next_out = dest;
|
317
|
-
comp_stream_.avail_out = (uInt)*destLen;
|
318
|
-
if ((uLong)comp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
|
319
|
-
|
320
|
-
if ( !first_chunk_ ) // only need to set up stream the first time through
|
321
|
-
return Z_OK;
|
322
|
-
|
323
|
-
if (comp_init_) { // we've already initted it
|
324
|
-
err = deflateReset(&comp_stream_);
|
325
|
-
if (err != Z_OK) {
|
326
|
-
LOG(WARNING) << "ERROR: Can't reset compress object; creating a new one";
|
327
|
-
deflateEnd(&comp_stream_);
|
328
|
-
comp_init_ = false;
|
329
|
-
}
|
330
|
-
}
|
331
|
-
if (!comp_init_) { // first use
|
332
|
-
comp_stream_.zalloc = (alloc_func)0;
|
333
|
-
comp_stream_.zfree = (free_func)0;
|
334
|
-
comp_stream_.opaque = (voidpf)0;
|
335
|
-
err = DeflateInit();
|
336
|
-
if (err != Z_OK) return err;
|
337
|
-
comp_init_ = true;
|
338
|
-
}
|
339
|
-
return Z_OK;
|
340
|
-
}
|
341
|
-
|
342
|
-
// In a perfect world we'd always have the full buffer to compress
|
343
|
-
// when the time came, and we could just call Compress(). Alas, we
|
344
|
-
// want to do chunked compression on our webserver. In this
|
345
|
-
// application, we compress the header, send it off, then compress the
|
346
|
-
// results, send them off, then compress the footer. Thus we need to
|
347
|
-
// use the chunked compression features of zlib.
|
348
|
-
int ZLib::CompressAtMostOrAll(Bytef *dest, uLongf *destLen,
|
349
|
-
const Bytef *source, uLong *sourceLen,
|
350
|
-
int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
|
351
|
-
int err;
|
352
|
-
|
353
|
-
if ( (err=CompressInit(dest, destLen, source, sourceLen)) != Z_OK )
|
354
|
-
return err;
|
355
|
-
|
356
|
-
// This is used to figure out how many bytes we wrote *this chunk*
|
357
|
-
int compressed_size = comp_stream_.total_out;
|
358
|
-
|
359
|
-
// Some setup happens only for the first chunk we compress in a run
|
360
|
-
if ( first_chunk_ ) {
|
361
|
-
first_chunk_ = false;
|
362
|
-
}
|
363
|
-
|
364
|
-
// flush_mode is Z_FINISH for all mode, Z_SYNC_FLUSH for incremental
|
365
|
-
// compression.
|
366
|
-
err = deflate(&comp_stream_, flush_mode);
|
367
|
-
|
368
|
-
*sourceLen = comp_stream_.avail_in;
|
369
|
-
|
370
|
-
if ((err == Z_STREAM_END || err == Z_OK)
|
371
|
-
&& comp_stream_.avail_in == 0
|
372
|
-
&& comp_stream_.avail_out != 0 ) {
|
373
|
-
// we processed everything ok and the output buffer was large enough.
|
374
|
-
;
|
375
|
-
} else if (err == Z_STREAM_END && comp_stream_.avail_in > 0) {
|
376
|
-
return Z_BUF_ERROR; // should never happen
|
377
|
-
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
|
378
|
-
// an error happened
|
379
|
-
CompressErrorInit();
|
380
|
-
return err;
|
381
|
-
} else if (comp_stream_.avail_out == 0) { // not enough space
|
382
|
-
err = Z_BUF_ERROR;
|
383
|
-
}
|
384
|
-
|
385
|
-
assert(err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR);
|
386
|
-
if (err == Z_STREAM_END)
|
387
|
-
err = Z_OK;
|
388
|
-
|
389
|
-
// update the crc and other metadata
|
390
|
-
compressed_size = comp_stream_.total_out - compressed_size; // delta
|
391
|
-
*destLen = compressed_size;
|
392
|
-
|
393
|
-
return err;
|
394
|
-
}
|
395
|
-
|
396
|
-
int ZLib::CompressChunkOrAll(Bytef *dest, uLongf *destLen,
|
397
|
-
const Bytef *source, uLong sourceLen,
|
398
|
-
int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
|
399
|
-
const int ret =
|
400
|
-
CompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
|
401
|
-
if (ret == Z_BUF_ERROR)
|
402
|
-
CompressErrorInit();
|
403
|
-
return ret;
|
404
|
-
}
|
405
|
-
|
406
|
-
// This routine only initializes the compression stream once. Thereafter, it
|
407
|
-
// just does a deflateReset on the stream, which should be faster.
|
408
|
-
int ZLib::Compress(Bytef *dest, uLongf *destLen,
|
409
|
-
const Bytef *source, uLong sourceLen) {
|
410
|
-
int err;
|
411
|
-
if ( (err=CompressChunkOrAll(dest, destLen, source, sourceLen,
|
412
|
-
Z_FINISH)) != Z_OK )
|
413
|
-
return err;
|
414
|
-
Reset(); // reset for next call to Compress
|
415
|
-
|
416
|
-
return Z_OK;
|
417
|
-
}
|
418
|
-
|
419
|
-
|
420
|
-
// --------- UNCOMPRESS MODE
|
421
|
-
|
422
|
-
int ZLib::InflateInit() {
|
423
|
-
return inflateInit2(&uncomp_stream_, MAX_WBITS);
|
424
|
-
}
|
425
|
-
|
426
|
-
// Initialization method to be called if we hit an error while
|
427
|
-
// uncompressing. On hitting an error, call this method before
|
428
|
-
// returning the error.
|
429
|
-
void ZLib::UncompressErrorInit() {
|
430
|
-
inflateEnd(&uncomp_stream_);
|
431
|
-
uncomp_init_ = false;
|
432
|
-
Reset();
|
433
|
-
}
|
434
|
-
|
435
|
-
int ZLib::UncompressInit(Bytef *dest, uLongf *destLen,
|
436
|
-
const Bytef *source, uLong *sourceLen) {
|
437
|
-
int err;
|
438
|
-
|
439
|
-
uncomp_stream_.next_in = (Bytef*)source;
|
440
|
-
uncomp_stream_.avail_in = (uInt)*sourceLen;
|
441
|
-
// Check for source > 64K on 16-bit machine:
|
442
|
-
if ((uLong)uncomp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
|
443
|
-
|
444
|
-
uncomp_stream_.next_out = dest;
|
445
|
-
uncomp_stream_.avail_out = (uInt)*destLen;
|
446
|
-
if ((uLong)uncomp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
|
447
|
-
|
448
|
-
if ( !first_chunk_ ) // only need to set up stream the first time through
|
449
|
-
return Z_OK;
|
450
|
-
|
451
|
-
if (uncomp_init_) { // we've already initted it
|
452
|
-
err = inflateReset(&uncomp_stream_);
|
453
|
-
if (err != Z_OK) {
|
454
|
-
LOG(WARNING)
|
455
|
-
<< "ERROR: Can't reset uncompress object; creating a new one";
|
456
|
-
UncompressErrorInit();
|
457
|
-
}
|
458
|
-
}
|
459
|
-
if (!uncomp_init_) {
|
460
|
-
uncomp_stream_.zalloc = (alloc_func)0;
|
461
|
-
uncomp_stream_.zfree = (free_func)0;
|
462
|
-
uncomp_stream_.opaque = (voidpf)0;
|
463
|
-
err = InflateInit();
|
464
|
-
if (err != Z_OK) return err;
|
465
|
-
uncomp_init_ = true;
|
466
|
-
}
|
467
|
-
return Z_OK;
|
468
|
-
}
|
469
|
-
|
470
|
-
// If you compressed your data a chunk at a time, with CompressChunk,
|
471
|
-
// you can uncompress it a chunk at a time with UncompressChunk.
|
472
|
-
// Only difference bewteen chunked and unchunked uncompression
|
473
|
-
// is the flush mode we use: Z_SYNC_FLUSH (chunked) or Z_FINISH (unchunked).
|
474
|
-
int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
|
475
|
-
const Bytef *source, uLong *sourceLen,
|
476
|
-
int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
|
477
|
-
int err = Z_OK;
|
478
|
-
|
479
|
-
if ( (err=UncompressInit(dest, destLen, source, sourceLen)) != Z_OK ) {
|
480
|
-
LOG(WARNING) << "UncompressInit: Error: " << err << " SourceLen: "
|
481
|
-
<< *sourceLen;
|
482
|
-
return err;
|
483
|
-
}
|
484
|
-
|
485
|
-
// This is used to figure out how many output bytes we wrote *this chunk*:
|
486
|
-
const uLong old_total_out = uncomp_stream_.total_out;
|
487
|
-
|
488
|
-
// This is used to figure out how many input bytes we read *this chunk*:
|
489
|
-
const uLong old_total_in = uncomp_stream_.total_in;
|
490
|
-
|
491
|
-
// Some setup happens only for the first chunk we compress in a run
|
492
|
-
if ( first_chunk_ ) {
|
493
|
-
first_chunk_ = false; // so we don't do this again
|
494
|
-
|
495
|
-
// For the first chunk *only* (to avoid infinite troubles), we let
|
496
|
-
// there be no actual data to uncompress. This sometimes triggers
|
497
|
-
// when the input is only the gzip header, say.
|
498
|
-
if ( *sourceLen == 0 ) {
|
499
|
-
*destLen = 0;
|
500
|
-
return Z_OK;
|
501
|
-
}
|
502
|
-
}
|
503
|
-
|
504
|
-
// We'll uncompress as much as we can. If we end OK great, otherwise
|
505
|
-
// if we get an error that seems to be the gzip footer, we store the
|
506
|
-
// gzip footer and return OK, otherwise we return the error.
|
507
|
-
|
508
|
-
// flush_mode is Z_SYNC_FLUSH for chunked mode, Z_FINISH for all mode.
|
509
|
-
err = inflate(&uncomp_stream_, flush_mode);
|
510
|
-
|
511
|
-
// Figure out how many bytes of the input zlib slurped up:
|
512
|
-
const uLong bytes_read = uncomp_stream_.total_in - old_total_in;
|
513
|
-
CHECK_LE(source + bytes_read, source + *sourceLen);
|
514
|
-
*sourceLen = uncomp_stream_.avail_in;
|
515
|
-
|
516
|
-
if ((err == Z_STREAM_END || err == Z_OK) // everything went ok
|
517
|
-
&& uncomp_stream_.avail_in == 0) { // and we read it all
|
518
|
-
;
|
519
|
-
} else if (err == Z_STREAM_END && uncomp_stream_.avail_in > 0) {
|
520
|
-
LOG(WARNING)
|
521
|
-
<< "UncompressChunkOrAll: Received some extra data, bytes total: "
|
522
|
-
<< uncomp_stream_.avail_in << " bytes: "
|
523
|
-
<< string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
524
|
-
min(int(uncomp_stream_.avail_in), 20));
|
525
|
-
UncompressErrorInit();
|
526
|
-
return Z_DATA_ERROR; // what's the extra data for?
|
527
|
-
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
|
528
|
-
// an error happened
|
529
|
-
LOG(WARNING) << "UncompressChunkOrAll: Error: " << err
|
530
|
-
<< " avail_out: " << uncomp_stream_.avail_out;
|
531
|
-
UncompressErrorInit();
|
532
|
-
return err;
|
533
|
-
} else if (uncomp_stream_.avail_out == 0) {
|
534
|
-
err = Z_BUF_ERROR;
|
535
|
-
}
|
536
|
-
|
537
|
-
assert(err == Z_OK || err == Z_BUF_ERROR || err == Z_STREAM_END);
|
538
|
-
if (err == Z_STREAM_END)
|
539
|
-
err = Z_OK;
|
540
|
-
|
541
|
-
*destLen = uncomp_stream_.total_out - old_total_out; // size for this call
|
542
|
-
|
543
|
-
return err;
|
544
|
-
}
|
545
|
-
|
546
|
-
int ZLib::UncompressChunkOrAll(Bytef *dest, uLongf *destLen,
|
547
|
-
const Bytef *source, uLong sourceLen,
|
548
|
-
int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
|
549
|
-
const int ret =
|
550
|
-
UncompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
|
551
|
-
if (ret == Z_BUF_ERROR)
|
552
|
-
UncompressErrorInit();
|
553
|
-
return ret;
|
554
|
-
}
|
555
|
-
|
556
|
-
int ZLib::UncompressAtMost(Bytef *dest, uLongf *destLen,
|
557
|
-
const Bytef *source, uLong *sourceLen) {
|
558
|
-
return UncompressAtMostOrAll(dest, destLen, source, sourceLen, Z_SYNC_FLUSH);
|
559
|
-
}
|
560
|
-
|
561
|
-
// We make sure we've uncompressed everything, that is, the current
|
562
|
-
// uncompress stream is at a compressed-buffer-EOF boundary. In gzip
|
563
|
-
// mode, we also check the gzip footer to make sure we pass the gzip
|
564
|
-
// consistency checks. We RETURN true iff both types of checks pass.
|
565
|
-
bool ZLib::UncompressChunkDone() {
|
566
|
-
assert(!first_chunk_ && uncomp_init_);
|
567
|
-
// Make sure we're at the end-of-compressed-data point. This means
|
568
|
-
// if we call inflate with Z_FINISH we won't consume any input or
|
569
|
-
// write any output
|
570
|
-
Bytef dummyin, dummyout;
|
571
|
-
uLongf dummylen = 0;
|
572
|
-
if ( UncompressChunkOrAll(&dummyout, &dummylen, &dummyin, 0, Z_FINISH)
|
573
|
-
!= Z_OK ) {
|
574
|
-
return false;
|
575
|
-
}
|
576
|
-
|
577
|
-
// Make sure that when we exit, we can start a new round of chunks later
|
578
|
-
Reset();
|
579
|
-
|
580
|
-
return true;
|
581
|
-
}
|
582
|
-
|
583
|
-
// Uncompresses the source buffer into the destination buffer.
|
584
|
-
// The destination buffer must be long enough to hold the entire
|
585
|
-
// decompressed contents.
|
586
|
-
//
|
587
|
-
// We only initialize the uncomp_stream once. Thereafter, we use
|
588
|
-
// inflateReset, which should be faster.
|
589
|
-
//
|
590
|
-
// Returns Z_OK on success, otherwise, it returns a zlib error code.
|
591
|
-
int ZLib::Uncompress(Bytef *dest, uLongf *destLen,
|
592
|
-
const Bytef *source, uLong sourceLen) {
|
593
|
-
int err;
|
594
|
-
if ( (err=UncompressChunkOrAll(dest, destLen, source, sourceLen,
|
595
|
-
Z_FINISH)) != Z_OK ) {
|
596
|
-
Reset(); // let us try to compress again
|
597
|
-
return err;
|
598
|
-
}
|
599
|
-
if ( !UncompressChunkDone() ) // calls Reset()
|
600
|
-
return Z_DATA_ERROR;
|
601
|
-
return Z_OK; // stream_end is ok
|
602
|
-
}
|
603
|
-
|
604
|
-
#endif // HAVE_LIBZ
|
605
|
-
|
606
|
-
} // namespace snappy
|