snappy 0.0.13 → 0.0.14

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -1
  3. data/lib/snappy/version.rb +1 -1
  4. data/vendor/snappy/AUTHORS +1 -0
  5. data/vendor/snappy/COPYING +54 -0
  6. data/vendor/snappy/ChangeLog +1916 -0
  7. data/vendor/snappy/Makefile.am +23 -0
  8. data/vendor/snappy/NEWS +128 -0
  9. data/vendor/snappy/README +135 -0
  10. data/vendor/snappy/autogen.sh +7 -0
  11. data/vendor/snappy/configure.ac +133 -0
  12. data/vendor/snappy/format_description.txt +110 -0
  13. data/vendor/snappy/framing_format.txt +135 -0
  14. data/vendor/snappy/m4/gtest.m4 +74 -0
  15. data/vendor/snappy/snappy-c.cc +90 -0
  16. data/vendor/snappy/snappy-c.h +138 -0
  17. data/vendor/snappy/snappy-internal.h +150 -0
  18. data/vendor/snappy/snappy-sinksource.cc +71 -0
  19. data/vendor/snappy/snappy-sinksource.h +137 -0
  20. data/vendor/snappy/snappy-stubs-internal.cc +42 -0
  21. data/vendor/snappy/snappy-stubs-internal.h +491 -0
  22. data/vendor/snappy/snappy-stubs-public.h.in +98 -0
  23. data/vendor/snappy/snappy-test.cc +606 -0
  24. data/vendor/snappy/snappy-test.h +582 -0
  25. data/vendor/snappy/snappy.cc +1306 -0
  26. data/vendor/snappy/snappy.h +184 -0
  27. data/vendor/snappy/snappy_unittest.cc +1355 -0
  28. data/vendor/snappy/testdata/alice29.txt +3609 -0
  29. data/vendor/snappy/testdata/asyoulik.txt +4122 -0
  30. data/vendor/snappy/testdata/baddata1.snappy +0 -0
  31. data/vendor/snappy/testdata/baddata2.snappy +0 -0
  32. data/vendor/snappy/testdata/baddata3.snappy +0 -0
  33. data/vendor/snappy/testdata/fireworks.jpeg +0 -0
  34. data/vendor/snappy/testdata/geo.protodata +0 -0
  35. data/vendor/snappy/testdata/html +1 -0
  36. data/vendor/snappy/testdata/html_x_4 +1 -0
  37. data/vendor/snappy/testdata/kppkn.gtb +0 -0
  38. data/vendor/snappy/testdata/lcet10.txt +7519 -0
  39. data/vendor/snappy/testdata/paper-100k.pdf +600 -2
  40. data/vendor/snappy/testdata/plrabn12.txt +10699 -0
  41. data/vendor/snappy/testdata/urls.10K +10000 -0
  42. metadata +40 -2
@@ -0,0 +1,98 @@
1
+ // Copyright 2011 Google Inc. All Rights Reserved.
2
+ // Author: sesse@google.com (Steinar H. Gunderson)
3
+ //
4
+ // Redistribution and use in source and binary forms, with or without
5
+ // modification, are permitted provided that the following conditions are
6
+ // met:
7
+ //
8
+ // * Redistributions of source code must retain the above copyright
9
+ // notice, this list of conditions and the following disclaimer.
10
+ // * Redistributions in binary form must reproduce the above
11
+ // copyright notice, this list of conditions and the following disclaimer
12
+ // in the documentation and/or other materials provided with the
13
+ // distribution.
14
+ // * Neither the name of Google Inc. nor the names of its
15
+ // contributors may be used to endorse or promote products derived from
16
+ // this software without specific prior written permission.
17
+ //
18
+ // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
+ // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
+ // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
+ // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
+ // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
+ // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
+ // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
+ // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
+ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
+ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
+ //
30
+ // Various type stubs for the open-source version of Snappy.
31
+ //
32
+ // This file cannot include config.h, as it is included from snappy.h,
33
+ // which is a public header. Instead, snappy-stubs-public.h is generated by
34
+ // from snappy-stubs-public.h.in at configure time.
35
+
36
+ #ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
37
+ #define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
38
+
39
+ #if @ac_cv_have_stdint_h@
40
+ #include <stdint.h>
41
+ #endif
42
+
43
+ #if @ac_cv_have_stddef_h@
44
+ #include <stddef.h>
45
+ #endif
46
+
47
+ #if @ac_cv_have_sys_uio_h@
48
+ #include <sys/uio.h>
49
+ #endif
50
+
51
+ #define SNAPPY_MAJOR @SNAPPY_MAJOR@
52
+ #define SNAPPY_MINOR @SNAPPY_MINOR@
53
+ #define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
54
+ #define SNAPPY_VERSION \
55
+ ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
56
+
57
+ #include <string>
58
+
59
+ namespace snappy {
60
+
61
+ #if @ac_cv_have_stdint_h@
62
+ typedef int8_t int8;
63
+ typedef uint8_t uint8;
64
+ typedef int16_t int16;
65
+ typedef uint16_t uint16;
66
+ typedef int32_t int32;
67
+ typedef uint32_t uint32;
68
+ typedef int64_t int64;
69
+ typedef uint64_t uint64;
70
+ #else
71
+ typedef signed char int8;
72
+ typedef unsigned char uint8;
73
+ typedef short int16;
74
+ typedef unsigned short uint16;
75
+ typedef int int32;
76
+ typedef unsigned int uint32;
77
+ typedef long long int64;
78
+ typedef unsigned long long uint64;
79
+ #endif
80
+
81
+ typedef std::string string;
82
+
83
+ #define DISALLOW_COPY_AND_ASSIGN(TypeName) \
84
+ TypeName(const TypeName&); \
85
+ void operator=(const TypeName&)
86
+
87
+ #if !@ac_cv_have_sys_uio_h@
88
+ // Windows does not have an iovec type, yet the concept is universally useful.
89
+ // It is simple to define it ourselves, so we put it inside our own namespace.
90
+ struct iovec {
91
+ void* iov_base;
92
+ size_t iov_len;
93
+ };
94
+ #endif
95
+
96
+ } // namespace snappy
97
+
98
+ #endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
@@ -0,0 +1,606 @@
1
+ // Copyright 2011 Google Inc. All Rights Reserved.
2
+ //
3
+ // Redistribution and use in source and binary forms, with or without
4
+ // modification, are permitted provided that the following conditions are
5
+ // met:
6
+ //
7
+ // * Redistributions of source code must retain the above copyright
8
+ // notice, this list of conditions and the following disclaimer.
9
+ // * Redistributions in binary form must reproduce the above
10
+ // copyright notice, this list of conditions and the following disclaimer
11
+ // in the documentation and/or other materials provided with the
12
+ // distribution.
13
+ // * Neither the name of Google Inc. nor the names of its
14
+ // contributors may be used to endorse or promote products derived from
15
+ // this software without specific prior written permission.
16
+ //
17
+ // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
+ // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
+ // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
+ // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21
+ // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
+ // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
+ // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24
+ // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25
+ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
+ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27
+ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
+ //
29
+ // Various stubs for the unit tests for the open-source version of Snappy.
30
+
31
+ #include "snappy-test.h"
32
+
33
+ #ifdef HAVE_WINDOWS_H
34
+ #define WIN32_LEAN_AND_MEAN
35
+ #include <windows.h>
36
+ #endif
37
+
38
+ #include <algorithm>
39
+
40
+ DEFINE_bool(run_microbenchmarks, true,
41
+ "Run microbenchmarks before doing anything else.");
42
+
43
+ namespace snappy {
44
+
45
+ string ReadTestDataFile(const string& base, size_t size_limit) {
46
+ string contents;
47
+ const char* srcdir = getenv("srcdir"); // This is set by Automake.
48
+ string prefix;
49
+ if (srcdir) {
50
+ prefix = string(srcdir) + "/";
51
+ }
52
+ file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
53
+ ).CheckSuccess();
54
+ if (size_limit > 0) {
55
+ contents = contents.substr(0, size_limit);
56
+ }
57
+ return contents;
58
+ }
59
+
60
+ string ReadTestDataFile(const string& base) {
61
+ return ReadTestDataFile(base, 0);
62
+ }
63
+
64
+ string StringPrintf(const char* format, ...) {
65
+ char buf[4096];
66
+ va_list ap;
67
+ va_start(ap, format);
68
+ vsnprintf(buf, sizeof(buf), format, ap);
69
+ va_end(ap);
70
+ return buf;
71
+ }
72
+
73
+ bool benchmark_running = false;
74
+ int64 benchmark_real_time_us = 0;
75
+ int64 benchmark_cpu_time_us = 0;
76
+ string *benchmark_label = NULL;
77
+ int64 benchmark_bytes_processed = 0;
78
+
79
+ void ResetBenchmarkTiming() {
80
+ benchmark_real_time_us = 0;
81
+ benchmark_cpu_time_us = 0;
82
+ }
83
+
84
+ #ifdef WIN32
85
+ LARGE_INTEGER benchmark_start_real;
86
+ FILETIME benchmark_start_cpu;
87
+ #else // WIN32
88
+ struct timeval benchmark_start_real;
89
+ struct rusage benchmark_start_cpu;
90
+ #endif // WIN32
91
+
92
+ void StartBenchmarkTiming() {
93
+ #ifdef WIN32
94
+ QueryPerformanceCounter(&benchmark_start_real);
95
+ FILETIME dummy;
96
+ CHECK(GetProcessTimes(
97
+ GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_start_cpu));
98
+ #else
99
+ gettimeofday(&benchmark_start_real, NULL);
100
+ if (getrusage(RUSAGE_SELF, &benchmark_start_cpu) == -1) {
101
+ perror("getrusage(RUSAGE_SELF)");
102
+ exit(1);
103
+ }
104
+ #endif
105
+ benchmark_running = true;
106
+ }
107
+
108
+ void StopBenchmarkTiming() {
109
+ if (!benchmark_running) {
110
+ return;
111
+ }
112
+
113
+ #ifdef WIN32
114
+ LARGE_INTEGER benchmark_stop_real;
115
+ LARGE_INTEGER benchmark_frequency;
116
+ QueryPerformanceCounter(&benchmark_stop_real);
117
+ QueryPerformanceFrequency(&benchmark_frequency);
118
+
119
+ double elapsed_real = static_cast<double>(
120
+ benchmark_stop_real.QuadPart - benchmark_start_real.QuadPart) /
121
+ benchmark_frequency.QuadPart;
122
+ benchmark_real_time_us += elapsed_real * 1e6 + 0.5;
123
+
124
+ FILETIME benchmark_stop_cpu, dummy;
125
+ CHECK(GetProcessTimes(
126
+ GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_stop_cpu));
127
+
128
+ ULARGE_INTEGER start_ulargeint;
129
+ start_ulargeint.LowPart = benchmark_start_cpu.dwLowDateTime;
130
+ start_ulargeint.HighPart = benchmark_start_cpu.dwHighDateTime;
131
+
132
+ ULARGE_INTEGER stop_ulargeint;
133
+ stop_ulargeint.LowPart = benchmark_stop_cpu.dwLowDateTime;
134
+ stop_ulargeint.HighPart = benchmark_stop_cpu.dwHighDateTime;
135
+
136
+ benchmark_cpu_time_us +=
137
+ (stop_ulargeint.QuadPart - start_ulargeint.QuadPart + 5) / 10;
138
+ #else // WIN32
139
+ struct timeval benchmark_stop_real;
140
+ gettimeofday(&benchmark_stop_real, NULL);
141
+ benchmark_real_time_us +=
142
+ 1000000 * (benchmark_stop_real.tv_sec - benchmark_start_real.tv_sec);
143
+ benchmark_real_time_us +=
144
+ (benchmark_stop_real.tv_usec - benchmark_start_real.tv_usec);
145
+
146
+ struct rusage benchmark_stop_cpu;
147
+ if (getrusage(RUSAGE_SELF, &benchmark_stop_cpu) == -1) {
148
+ perror("getrusage(RUSAGE_SELF)");
149
+ exit(1);
150
+ }
151
+ benchmark_cpu_time_us += 1000000 * (benchmark_stop_cpu.ru_utime.tv_sec -
152
+ benchmark_start_cpu.ru_utime.tv_sec);
153
+ benchmark_cpu_time_us += (benchmark_stop_cpu.ru_utime.tv_usec -
154
+ benchmark_start_cpu.ru_utime.tv_usec);
155
+ #endif // WIN32
156
+
157
+ benchmark_running = false;
158
+ }
159
+
160
+ void SetBenchmarkLabel(const string& str) {
161
+ if (benchmark_label) {
162
+ delete benchmark_label;
163
+ }
164
+ benchmark_label = new string(str);
165
+ }
166
+
167
+ void SetBenchmarkBytesProcessed(int64 bytes) {
168
+ benchmark_bytes_processed = bytes;
169
+ }
170
+
171
+ struct BenchmarkRun {
172
+ int64 real_time_us;
173
+ int64 cpu_time_us;
174
+ };
175
+
176
+ struct BenchmarkCompareCPUTime {
177
+ bool operator() (const BenchmarkRun& a, const BenchmarkRun& b) const {
178
+ return a.cpu_time_us < b.cpu_time_us;
179
+ }
180
+ };
181
+
182
+ void Benchmark::Run() {
183
+ for (int test_case_num = start_; test_case_num <= stop_; ++test_case_num) {
184
+ // Run a few iterations first to find out approximately how fast
185
+ // the benchmark is.
186
+ const int kCalibrateIterations = 100;
187
+ ResetBenchmarkTiming();
188
+ StartBenchmarkTiming();
189
+ (*function_)(kCalibrateIterations, test_case_num);
190
+ StopBenchmarkTiming();
191
+
192
+ // Let each test case run for about 200ms, but at least as many
193
+ // as we used to calibrate.
194
+ // Run five times and pick the median.
195
+ const int kNumRuns = 5;
196
+ const int kMedianPos = kNumRuns / 2;
197
+ int num_iterations = 0;
198
+ if (benchmark_real_time_us > 0) {
199
+ num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
200
+ }
201
+ num_iterations = max(num_iterations, kCalibrateIterations);
202
+ BenchmarkRun benchmark_runs[kNumRuns];
203
+
204
+ for (int run = 0; run < kNumRuns; ++run) {
205
+ ResetBenchmarkTiming();
206
+ StartBenchmarkTiming();
207
+ (*function_)(num_iterations, test_case_num);
208
+ StopBenchmarkTiming();
209
+
210
+ benchmark_runs[run].real_time_us = benchmark_real_time_us;
211
+ benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us;
212
+ }
213
+
214
+ string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
215
+ string human_readable_speed;
216
+
217
+ nth_element(benchmark_runs,
218
+ benchmark_runs + kMedianPos,
219
+ benchmark_runs + kNumRuns,
220
+ BenchmarkCompareCPUTime());
221
+ int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
222
+ int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
223
+ if (cpu_time_us <= 0) {
224
+ human_readable_speed = "?";
225
+ } else {
226
+ int64 bytes_per_second =
227
+ benchmark_bytes_processed * 1000000 / cpu_time_us;
228
+ if (bytes_per_second < 1024) {
229
+ human_readable_speed = StringPrintf("%dB/s", bytes_per_second);
230
+ } else if (bytes_per_second < 1024 * 1024) {
231
+ human_readable_speed = StringPrintf(
232
+ "%.1fkB/s", bytes_per_second / 1024.0f);
233
+ } else if (bytes_per_second < 1024 * 1024 * 1024) {
234
+ human_readable_speed = StringPrintf(
235
+ "%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
236
+ } else {
237
+ human_readable_speed = StringPrintf(
238
+ "%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
239
+ }
240
+ }
241
+
242
+ fprintf(stderr,
243
+ #ifdef WIN32
244
+ "%-18s %10I64d %10I64d %10d %s %s\n",
245
+ #else
246
+ "%-18s %10lld %10lld %10d %s %s\n",
247
+ #endif
248
+ heading.c_str(),
249
+ static_cast<long long>(real_time_us * 1000 / num_iterations),
250
+ static_cast<long long>(cpu_time_us * 1000 / num_iterations),
251
+ num_iterations,
252
+ human_readable_speed.c_str(),
253
+ benchmark_label->c_str());
254
+ }
255
+ }
256
+
257
+ #ifdef HAVE_LIBZ
258
+
259
+ ZLib::ZLib()
260
+ : comp_init_(false),
261
+ uncomp_init_(false) {
262
+ Reinit();
263
+ }
264
+
265
+ ZLib::~ZLib() {
266
+ if (comp_init_) { deflateEnd(&comp_stream_); }
267
+ if (uncomp_init_) { inflateEnd(&uncomp_stream_); }
268
+ }
269
+
270
+ void ZLib::Reinit() {
271
+ compression_level_ = Z_DEFAULT_COMPRESSION;
272
+ window_bits_ = MAX_WBITS;
273
+ mem_level_ = 8; // DEF_MEM_LEVEL
274
+ if (comp_init_) {
275
+ deflateEnd(&comp_stream_);
276
+ comp_init_ = false;
277
+ }
278
+ if (uncomp_init_) {
279
+ inflateEnd(&uncomp_stream_);
280
+ uncomp_init_ = false;
281
+ }
282
+ first_chunk_ = true;
283
+ }
284
+
285
+ void ZLib::Reset() {
286
+ first_chunk_ = true;
287
+ }
288
+
289
+ // --------- COMPRESS MODE
290
+
291
+ // Initialization method to be called if we hit an error while
292
+ // compressing. On hitting an error, call this method before returning
293
+ // the error.
294
+ void ZLib::CompressErrorInit() {
295
+ deflateEnd(&comp_stream_);
296
+ comp_init_ = false;
297
+ Reset();
298
+ }
299
+
300
+ int ZLib::DeflateInit() {
301
+ return deflateInit2(&comp_stream_,
302
+ compression_level_,
303
+ Z_DEFLATED,
304
+ window_bits_,
305
+ mem_level_,
306
+ Z_DEFAULT_STRATEGY);
307
+ }
308
+
309
+ int ZLib::CompressInit(Bytef *dest, uLongf *destLen,
310
+ const Bytef *source, uLong *sourceLen) {
311
+ int err;
312
+
313
+ comp_stream_.next_in = (Bytef*)source;
314
+ comp_stream_.avail_in = (uInt)*sourceLen;
315
+ if ((uLong)comp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
316
+ comp_stream_.next_out = dest;
317
+ comp_stream_.avail_out = (uInt)*destLen;
318
+ if ((uLong)comp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
319
+
320
+ if ( !first_chunk_ ) // only need to set up stream the first time through
321
+ return Z_OK;
322
+
323
+ if (comp_init_) { // we've already initted it
324
+ err = deflateReset(&comp_stream_);
325
+ if (err != Z_OK) {
326
+ LOG(WARNING) << "ERROR: Can't reset compress object; creating a new one";
327
+ deflateEnd(&comp_stream_);
328
+ comp_init_ = false;
329
+ }
330
+ }
331
+ if (!comp_init_) { // first use
332
+ comp_stream_.zalloc = (alloc_func)0;
333
+ comp_stream_.zfree = (free_func)0;
334
+ comp_stream_.opaque = (voidpf)0;
335
+ err = DeflateInit();
336
+ if (err != Z_OK) return err;
337
+ comp_init_ = true;
338
+ }
339
+ return Z_OK;
340
+ }
341
+
342
+ // In a perfect world we'd always have the full buffer to compress
343
+ // when the time came, and we could just call Compress(). Alas, we
344
+ // want to do chunked compression on our webserver. In this
345
+ // application, we compress the header, send it off, then compress the
346
+ // results, send them off, then compress the footer. Thus we need to
347
+ // use the chunked compression features of zlib.
348
+ int ZLib::CompressAtMostOrAll(Bytef *dest, uLongf *destLen,
349
+ const Bytef *source, uLong *sourceLen,
350
+ int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
351
+ int err;
352
+
353
+ if ( (err=CompressInit(dest, destLen, source, sourceLen)) != Z_OK )
354
+ return err;
355
+
356
+ // This is used to figure out how many bytes we wrote *this chunk*
357
+ int compressed_size = comp_stream_.total_out;
358
+
359
+ // Some setup happens only for the first chunk we compress in a run
360
+ if ( first_chunk_ ) {
361
+ first_chunk_ = false;
362
+ }
363
+
364
+ // flush_mode is Z_FINISH for all mode, Z_SYNC_FLUSH for incremental
365
+ // compression.
366
+ err = deflate(&comp_stream_, flush_mode);
367
+
368
+ *sourceLen = comp_stream_.avail_in;
369
+
370
+ if ((err == Z_STREAM_END || err == Z_OK)
371
+ && comp_stream_.avail_in == 0
372
+ && comp_stream_.avail_out != 0 ) {
373
+ // we processed everything ok and the output buffer was large enough.
374
+ ;
375
+ } else if (err == Z_STREAM_END && comp_stream_.avail_in > 0) {
376
+ return Z_BUF_ERROR; // should never happen
377
+ } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
378
+ // an error happened
379
+ CompressErrorInit();
380
+ return err;
381
+ } else if (comp_stream_.avail_out == 0) { // not enough space
382
+ err = Z_BUF_ERROR;
383
+ }
384
+
385
+ assert(err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR);
386
+ if (err == Z_STREAM_END)
387
+ err = Z_OK;
388
+
389
+ // update the crc and other metadata
390
+ compressed_size = comp_stream_.total_out - compressed_size; // delta
391
+ *destLen = compressed_size;
392
+
393
+ return err;
394
+ }
395
+
396
+ int ZLib::CompressChunkOrAll(Bytef *dest, uLongf *destLen,
397
+ const Bytef *source, uLong sourceLen,
398
+ int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
399
+ const int ret =
400
+ CompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
401
+ if (ret == Z_BUF_ERROR)
402
+ CompressErrorInit();
403
+ return ret;
404
+ }
405
+
406
+ // This routine only initializes the compression stream once. Thereafter, it
407
+ // just does a deflateReset on the stream, which should be faster.
408
+ int ZLib::Compress(Bytef *dest, uLongf *destLen,
409
+ const Bytef *source, uLong sourceLen) {
410
+ int err;
411
+ if ( (err=CompressChunkOrAll(dest, destLen, source, sourceLen,
412
+ Z_FINISH)) != Z_OK )
413
+ return err;
414
+ Reset(); // reset for next call to Compress
415
+
416
+ return Z_OK;
417
+ }
418
+
419
+
420
+ // --------- UNCOMPRESS MODE
421
+
422
+ int ZLib::InflateInit() {
423
+ return inflateInit2(&uncomp_stream_, MAX_WBITS);
424
+ }
425
+
426
+ // Initialization method to be called if we hit an error while
427
+ // uncompressing. On hitting an error, call this method before
428
+ // returning the error.
429
+ void ZLib::UncompressErrorInit() {
430
+ inflateEnd(&uncomp_stream_);
431
+ uncomp_init_ = false;
432
+ Reset();
433
+ }
434
+
435
+ int ZLib::UncompressInit(Bytef *dest, uLongf *destLen,
436
+ const Bytef *source, uLong *sourceLen) {
437
+ int err;
438
+
439
+ uncomp_stream_.next_in = (Bytef*)source;
440
+ uncomp_stream_.avail_in = (uInt)*sourceLen;
441
+ // Check for source > 64K on 16-bit machine:
442
+ if ((uLong)uncomp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
443
+
444
+ uncomp_stream_.next_out = dest;
445
+ uncomp_stream_.avail_out = (uInt)*destLen;
446
+ if ((uLong)uncomp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
447
+
448
+ if ( !first_chunk_ ) // only need to set up stream the first time through
449
+ return Z_OK;
450
+
451
+ if (uncomp_init_) { // we've already initted it
452
+ err = inflateReset(&uncomp_stream_);
453
+ if (err != Z_OK) {
454
+ LOG(WARNING)
455
+ << "ERROR: Can't reset uncompress object; creating a new one";
456
+ UncompressErrorInit();
457
+ }
458
+ }
459
+ if (!uncomp_init_) {
460
+ uncomp_stream_.zalloc = (alloc_func)0;
461
+ uncomp_stream_.zfree = (free_func)0;
462
+ uncomp_stream_.opaque = (voidpf)0;
463
+ err = InflateInit();
464
+ if (err != Z_OK) return err;
465
+ uncomp_init_ = true;
466
+ }
467
+ return Z_OK;
468
+ }
469
+
470
+ // If you compressed your data a chunk at a time, with CompressChunk,
471
+ // you can uncompress it a chunk at a time with UncompressChunk.
472
+ // Only difference bewteen chunked and unchunked uncompression
473
+ // is the flush mode we use: Z_SYNC_FLUSH (chunked) or Z_FINISH (unchunked).
474
+ int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
475
+ const Bytef *source, uLong *sourceLen,
476
+ int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
477
+ int err = Z_OK;
478
+
479
+ if ( (err=UncompressInit(dest, destLen, source, sourceLen)) != Z_OK ) {
480
+ LOG(WARNING) << "UncompressInit: Error: " << err << " SourceLen: "
481
+ << *sourceLen;
482
+ return err;
483
+ }
484
+
485
+ // This is used to figure out how many output bytes we wrote *this chunk*:
486
+ const uLong old_total_out = uncomp_stream_.total_out;
487
+
488
+ // This is used to figure out how many input bytes we read *this chunk*:
489
+ const uLong old_total_in = uncomp_stream_.total_in;
490
+
491
+ // Some setup happens only for the first chunk we compress in a run
492
+ if ( first_chunk_ ) {
493
+ first_chunk_ = false; // so we don't do this again
494
+
495
+ // For the first chunk *only* (to avoid infinite troubles), we let
496
+ // there be no actual data to uncompress. This sometimes triggers
497
+ // when the input is only the gzip header, say.
498
+ if ( *sourceLen == 0 ) {
499
+ *destLen = 0;
500
+ return Z_OK;
501
+ }
502
+ }
503
+
504
+ // We'll uncompress as much as we can. If we end OK great, otherwise
505
+ // if we get an error that seems to be the gzip footer, we store the
506
+ // gzip footer and return OK, otherwise we return the error.
507
+
508
+ // flush_mode is Z_SYNC_FLUSH for chunked mode, Z_FINISH for all mode.
509
+ err = inflate(&uncomp_stream_, flush_mode);
510
+
511
+ // Figure out how many bytes of the input zlib slurped up:
512
+ const uLong bytes_read = uncomp_stream_.total_in - old_total_in;
513
+ CHECK_LE(source + bytes_read, source + *sourceLen);
514
+ *sourceLen = uncomp_stream_.avail_in;
515
+
516
+ if ((err == Z_STREAM_END || err == Z_OK) // everything went ok
517
+ && uncomp_stream_.avail_in == 0) { // and we read it all
518
+ ;
519
+ } else if (err == Z_STREAM_END && uncomp_stream_.avail_in > 0) {
520
+ LOG(WARNING)
521
+ << "UncompressChunkOrAll: Received some extra data, bytes total: "
522
+ << uncomp_stream_.avail_in << " bytes: "
523
+ << string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
524
+ min(int(uncomp_stream_.avail_in), 20));
525
+ UncompressErrorInit();
526
+ return Z_DATA_ERROR; // what's the extra data for?
527
+ } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
528
+ // an error happened
529
+ LOG(WARNING) << "UncompressChunkOrAll: Error: " << err
530
+ << " avail_out: " << uncomp_stream_.avail_out;
531
+ UncompressErrorInit();
532
+ return err;
533
+ } else if (uncomp_stream_.avail_out == 0) {
534
+ err = Z_BUF_ERROR;
535
+ }
536
+
537
+ assert(err == Z_OK || err == Z_BUF_ERROR || err == Z_STREAM_END);
538
+ if (err == Z_STREAM_END)
539
+ err = Z_OK;
540
+
541
+ *destLen = uncomp_stream_.total_out - old_total_out; // size for this call
542
+
543
+ return err;
544
+ }
545
+
546
+ int ZLib::UncompressChunkOrAll(Bytef *dest, uLongf *destLen,
547
+ const Bytef *source, uLong sourceLen,
548
+ int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
549
+ const int ret =
550
+ UncompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
551
+ if (ret == Z_BUF_ERROR)
552
+ UncompressErrorInit();
553
+ return ret;
554
+ }
555
+
556
+ int ZLib::UncompressAtMost(Bytef *dest, uLongf *destLen,
557
+ const Bytef *source, uLong *sourceLen) {
558
+ return UncompressAtMostOrAll(dest, destLen, source, sourceLen, Z_SYNC_FLUSH);
559
+ }
560
+
561
+ // We make sure we've uncompressed everything, that is, the current
562
+ // uncompress stream is at a compressed-buffer-EOF boundary. In gzip
563
+ // mode, we also check the gzip footer to make sure we pass the gzip
564
+ // consistency checks. We RETURN true iff both types of checks pass.
565
+ bool ZLib::UncompressChunkDone() {
566
+ assert(!first_chunk_ && uncomp_init_);
567
+ // Make sure we're at the end-of-compressed-data point. This means
568
+ // if we call inflate with Z_FINISH we won't consume any input or
569
+ // write any output
570
+ Bytef dummyin, dummyout;
571
+ uLongf dummylen = 0;
572
+ if ( UncompressChunkOrAll(&dummyout, &dummylen, &dummyin, 0, Z_FINISH)
573
+ != Z_OK ) {
574
+ return false;
575
+ }
576
+
577
+ // Make sure that when we exit, we can start a new round of chunks later
578
+ Reset();
579
+
580
+ return true;
581
+ }
582
+
583
+ // Uncompresses the source buffer into the destination buffer.
584
+ // The destination buffer must be long enough to hold the entire
585
+ // decompressed contents.
586
+ //
587
+ // We only initialize the uncomp_stream once. Thereafter, we use
588
+ // inflateReset, which should be faster.
589
+ //
590
+ // Returns Z_OK on success, otherwise, it returns a zlib error code.
591
+ int ZLib::Uncompress(Bytef *dest, uLongf *destLen,
592
+ const Bytef *source, uLong sourceLen) {
593
+ int err;
594
+ if ( (err=UncompressChunkOrAll(dest, destLen, source, sourceLen,
595
+ Z_FINISH)) != Z_OK ) {
596
+ Reset(); // let us try to compress again
597
+ return err;
598
+ }
599
+ if ( !UncompressChunkDone() ) // calls Reset()
600
+ return Z_DATA_ERROR;
601
+ return Z_OK; // stream_end is ok
602
+ }
603
+
604
+ #endif // HAVE_LIBZ
605
+
606
+ } // namespace snappy