snappy 0.0.12 → 0.0.13

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/lib/snappy/version.rb +1 -1
  3. data/lib/snappy/writer.rb +6 -0
  4. metadata +3 -41
  5. data/vendor/snappy/AUTHORS +0 -1
  6. data/vendor/snappy/COPYING +0 -54
  7. data/vendor/snappy/ChangeLog +0 -1916
  8. data/vendor/snappy/Makefile.am +0 -23
  9. data/vendor/snappy/NEWS +0 -128
  10. data/vendor/snappy/README +0 -135
  11. data/vendor/snappy/autogen.sh +0 -7
  12. data/vendor/snappy/configure.ac +0 -133
  13. data/vendor/snappy/format_description.txt +0 -110
  14. data/vendor/snappy/framing_format.txt +0 -135
  15. data/vendor/snappy/m4/gtest.m4 +0 -74
  16. data/vendor/snappy/snappy-c.cc +0 -90
  17. data/vendor/snappy/snappy-c.h +0 -138
  18. data/vendor/snappy/snappy-internal.h +0 -150
  19. data/vendor/snappy/snappy-sinksource.cc +0 -71
  20. data/vendor/snappy/snappy-sinksource.h +0 -137
  21. data/vendor/snappy/snappy-stubs-internal.cc +0 -42
  22. data/vendor/snappy/snappy-stubs-internal.h +0 -491
  23. data/vendor/snappy/snappy-stubs-public.h.in +0 -98
  24. data/vendor/snappy/snappy-test.cc +0 -606
  25. data/vendor/snappy/snappy-test.h +0 -582
  26. data/vendor/snappy/snappy.cc +0 -1306
  27. data/vendor/snappy/snappy.h +0 -184
  28. data/vendor/snappy/snappy_unittest.cc +0 -1355
  29. data/vendor/snappy/testdata/alice29.txt +0 -3609
  30. data/vendor/snappy/testdata/asyoulik.txt +0 -4122
  31. data/vendor/snappy/testdata/baddata1.snappy +0 -0
  32. data/vendor/snappy/testdata/baddata2.snappy +0 -0
  33. data/vendor/snappy/testdata/baddata3.snappy +0 -0
  34. data/vendor/snappy/testdata/fireworks.jpeg +0 -0
  35. data/vendor/snappy/testdata/geo.protodata +0 -0
  36. data/vendor/snappy/testdata/html +0 -1
  37. data/vendor/snappy/testdata/html_x_4 +0 -1
  38. data/vendor/snappy/testdata/kppkn.gtb +0 -0
  39. data/vendor/snappy/testdata/lcet10.txt +0 -7519
  40. data/vendor/snappy/testdata/paper-100k.pdf +2 -600
  41. data/vendor/snappy/testdata/plrabn12.txt +0 -10699
  42. data/vendor/snappy/testdata/urls.10K +0 -10000
@@ -1,98 +0,0 @@
1
- // Copyright 2011 Google Inc. All Rights Reserved.
2
- // Author: sesse@google.com (Steinar H. Gunderson)
3
- //
4
- // Redistribution and use in source and binary forms, with or without
5
- // modification, are permitted provided that the following conditions are
6
- // met:
7
- //
8
- // * Redistributions of source code must retain the above copyright
9
- // notice, this list of conditions and the following disclaimer.
10
- // * Redistributions in binary form must reproduce the above
11
- // copyright notice, this list of conditions and the following disclaimer
12
- // in the documentation and/or other materials provided with the
13
- // distribution.
14
- // * Neither the name of Google Inc. nor the names of its
15
- // contributors may be used to endorse or promote products derived from
16
- // this software without specific prior written permission.
17
- //
18
- // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
- // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
- // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
- // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
- // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
- // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
- // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
- // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
- // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
- // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
- // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
- //
30
- // Various type stubs for the open-source version of Snappy.
31
- //
32
- // This file cannot include config.h, as it is included from snappy.h,
33
- // which is a public header. Instead, snappy-stubs-public.h is generated by
34
- // from snappy-stubs-public.h.in at configure time.
35
-
36
- #ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
37
- #define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
38
-
39
- #if @ac_cv_have_stdint_h@
40
- #include <stdint.h>
41
- #endif
42
-
43
- #if @ac_cv_have_stddef_h@
44
- #include <stddef.h>
45
- #endif
46
-
47
- #if @ac_cv_have_sys_uio_h@
48
- #include <sys/uio.h>
49
- #endif
50
-
51
- #define SNAPPY_MAJOR @SNAPPY_MAJOR@
52
- #define SNAPPY_MINOR @SNAPPY_MINOR@
53
- #define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
54
- #define SNAPPY_VERSION \
55
- ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
56
-
57
- #include <string>
58
-
59
- namespace snappy {
60
-
61
- #if @ac_cv_have_stdint_h@
62
- typedef int8_t int8;
63
- typedef uint8_t uint8;
64
- typedef int16_t int16;
65
- typedef uint16_t uint16;
66
- typedef int32_t int32;
67
- typedef uint32_t uint32;
68
- typedef int64_t int64;
69
- typedef uint64_t uint64;
70
- #else
71
- typedef signed char int8;
72
- typedef unsigned char uint8;
73
- typedef short int16;
74
- typedef unsigned short uint16;
75
- typedef int int32;
76
- typedef unsigned int uint32;
77
- typedef long long int64;
78
- typedef unsigned long long uint64;
79
- #endif
80
-
81
- typedef std::string string;
82
-
83
- #define DISALLOW_COPY_AND_ASSIGN(TypeName) \
84
- TypeName(const TypeName&); \
85
- void operator=(const TypeName&)
86
-
87
- #if !@ac_cv_have_sys_uio_h@
88
- // Windows does not have an iovec type, yet the concept is universally useful.
89
- // It is simple to define it ourselves, so we put it inside our own namespace.
90
- struct iovec {
91
- void* iov_base;
92
- size_t iov_len;
93
- };
94
- #endif
95
-
96
- } // namespace snappy
97
-
98
- #endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
@@ -1,606 +0,0 @@
1
- // Copyright 2011 Google Inc. All Rights Reserved.
2
- //
3
- // Redistribution and use in source and binary forms, with or without
4
- // modification, are permitted provided that the following conditions are
5
- // met:
6
- //
7
- // * Redistributions of source code must retain the above copyright
8
- // notice, this list of conditions and the following disclaimer.
9
- // * Redistributions in binary form must reproduce the above
10
- // copyright notice, this list of conditions and the following disclaimer
11
- // in the documentation and/or other materials provided with the
12
- // distribution.
13
- // * Neither the name of Google Inc. nor the names of its
14
- // contributors may be used to endorse or promote products derived from
15
- // this software without specific prior written permission.
16
- //
17
- // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
- // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
- // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
- // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21
- // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
- // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
- // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24
- // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25
- // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
- // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27
- // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
- //
29
- // Various stubs for the unit tests for the open-source version of Snappy.
30
-
31
- #include "snappy-test.h"
32
-
33
- #ifdef HAVE_WINDOWS_H
34
- #define WIN32_LEAN_AND_MEAN
35
- #include <windows.h>
36
- #endif
37
-
38
- #include <algorithm>
39
-
40
- DEFINE_bool(run_microbenchmarks, true,
41
- "Run microbenchmarks before doing anything else.");
42
-
43
- namespace snappy {
44
-
45
- string ReadTestDataFile(const string& base, size_t size_limit) {
46
- string contents;
47
- const char* srcdir = getenv("srcdir"); // This is set by Automake.
48
- string prefix;
49
- if (srcdir) {
50
- prefix = string(srcdir) + "/";
51
- }
52
- file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
53
- ).CheckSuccess();
54
- if (size_limit > 0) {
55
- contents = contents.substr(0, size_limit);
56
- }
57
- return contents;
58
- }
59
-
60
- string ReadTestDataFile(const string& base) {
61
- return ReadTestDataFile(base, 0);
62
- }
63
-
64
- string StringPrintf(const char* format, ...) {
65
- char buf[4096];
66
- va_list ap;
67
- va_start(ap, format);
68
- vsnprintf(buf, sizeof(buf), format, ap);
69
- va_end(ap);
70
- return buf;
71
- }
72
-
73
- bool benchmark_running = false;
74
- int64 benchmark_real_time_us = 0;
75
- int64 benchmark_cpu_time_us = 0;
76
- string *benchmark_label = NULL;
77
- int64 benchmark_bytes_processed = 0;
78
-
79
- void ResetBenchmarkTiming() {
80
- benchmark_real_time_us = 0;
81
- benchmark_cpu_time_us = 0;
82
- }
83
-
84
- #ifdef WIN32
85
- LARGE_INTEGER benchmark_start_real;
86
- FILETIME benchmark_start_cpu;
87
- #else // WIN32
88
- struct timeval benchmark_start_real;
89
- struct rusage benchmark_start_cpu;
90
- #endif // WIN32
91
-
92
- void StartBenchmarkTiming() {
93
- #ifdef WIN32
94
- QueryPerformanceCounter(&benchmark_start_real);
95
- FILETIME dummy;
96
- CHECK(GetProcessTimes(
97
- GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_start_cpu));
98
- #else
99
- gettimeofday(&benchmark_start_real, NULL);
100
- if (getrusage(RUSAGE_SELF, &benchmark_start_cpu) == -1) {
101
- perror("getrusage(RUSAGE_SELF)");
102
- exit(1);
103
- }
104
- #endif
105
- benchmark_running = true;
106
- }
107
-
108
- void StopBenchmarkTiming() {
109
- if (!benchmark_running) {
110
- return;
111
- }
112
-
113
- #ifdef WIN32
114
- LARGE_INTEGER benchmark_stop_real;
115
- LARGE_INTEGER benchmark_frequency;
116
- QueryPerformanceCounter(&benchmark_stop_real);
117
- QueryPerformanceFrequency(&benchmark_frequency);
118
-
119
- double elapsed_real = static_cast<double>(
120
- benchmark_stop_real.QuadPart - benchmark_start_real.QuadPart) /
121
- benchmark_frequency.QuadPart;
122
- benchmark_real_time_us += elapsed_real * 1e6 + 0.5;
123
-
124
- FILETIME benchmark_stop_cpu, dummy;
125
- CHECK(GetProcessTimes(
126
- GetCurrentProcess(), &dummy, &dummy, &dummy, &benchmark_stop_cpu));
127
-
128
- ULARGE_INTEGER start_ulargeint;
129
- start_ulargeint.LowPart = benchmark_start_cpu.dwLowDateTime;
130
- start_ulargeint.HighPart = benchmark_start_cpu.dwHighDateTime;
131
-
132
- ULARGE_INTEGER stop_ulargeint;
133
- stop_ulargeint.LowPart = benchmark_stop_cpu.dwLowDateTime;
134
- stop_ulargeint.HighPart = benchmark_stop_cpu.dwHighDateTime;
135
-
136
- benchmark_cpu_time_us +=
137
- (stop_ulargeint.QuadPart - start_ulargeint.QuadPart + 5) / 10;
138
- #else // WIN32
139
- struct timeval benchmark_stop_real;
140
- gettimeofday(&benchmark_stop_real, NULL);
141
- benchmark_real_time_us +=
142
- 1000000 * (benchmark_stop_real.tv_sec - benchmark_start_real.tv_sec);
143
- benchmark_real_time_us +=
144
- (benchmark_stop_real.tv_usec - benchmark_start_real.tv_usec);
145
-
146
- struct rusage benchmark_stop_cpu;
147
- if (getrusage(RUSAGE_SELF, &benchmark_stop_cpu) == -1) {
148
- perror("getrusage(RUSAGE_SELF)");
149
- exit(1);
150
- }
151
- benchmark_cpu_time_us += 1000000 * (benchmark_stop_cpu.ru_utime.tv_sec -
152
- benchmark_start_cpu.ru_utime.tv_sec);
153
- benchmark_cpu_time_us += (benchmark_stop_cpu.ru_utime.tv_usec -
154
- benchmark_start_cpu.ru_utime.tv_usec);
155
- #endif // WIN32
156
-
157
- benchmark_running = false;
158
- }
159
-
160
- void SetBenchmarkLabel(const string& str) {
161
- if (benchmark_label) {
162
- delete benchmark_label;
163
- }
164
- benchmark_label = new string(str);
165
- }
166
-
167
- void SetBenchmarkBytesProcessed(int64 bytes) {
168
- benchmark_bytes_processed = bytes;
169
- }
170
-
171
- struct BenchmarkRun {
172
- int64 real_time_us;
173
- int64 cpu_time_us;
174
- };
175
-
176
- struct BenchmarkCompareCPUTime {
177
- bool operator() (const BenchmarkRun& a, const BenchmarkRun& b) const {
178
- return a.cpu_time_us < b.cpu_time_us;
179
- }
180
- };
181
-
182
- void Benchmark::Run() {
183
- for (int test_case_num = start_; test_case_num <= stop_; ++test_case_num) {
184
- // Run a few iterations first to find out approximately how fast
185
- // the benchmark is.
186
- const int kCalibrateIterations = 100;
187
- ResetBenchmarkTiming();
188
- StartBenchmarkTiming();
189
- (*function_)(kCalibrateIterations, test_case_num);
190
- StopBenchmarkTiming();
191
-
192
- // Let each test case run for about 200ms, but at least as many
193
- // as we used to calibrate.
194
- // Run five times and pick the median.
195
- const int kNumRuns = 5;
196
- const int kMedianPos = kNumRuns / 2;
197
- int num_iterations = 0;
198
- if (benchmark_real_time_us > 0) {
199
- num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
200
- }
201
- num_iterations = max(num_iterations, kCalibrateIterations);
202
- BenchmarkRun benchmark_runs[kNumRuns];
203
-
204
- for (int run = 0; run < kNumRuns; ++run) {
205
- ResetBenchmarkTiming();
206
- StartBenchmarkTiming();
207
- (*function_)(num_iterations, test_case_num);
208
- StopBenchmarkTiming();
209
-
210
- benchmark_runs[run].real_time_us = benchmark_real_time_us;
211
- benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us;
212
- }
213
-
214
- string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
215
- string human_readable_speed;
216
-
217
- nth_element(benchmark_runs,
218
- benchmark_runs + kMedianPos,
219
- benchmark_runs + kNumRuns,
220
- BenchmarkCompareCPUTime());
221
- int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
222
- int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
223
- if (cpu_time_us <= 0) {
224
- human_readable_speed = "?";
225
- } else {
226
- int64 bytes_per_second =
227
- benchmark_bytes_processed * 1000000 / cpu_time_us;
228
- if (bytes_per_second < 1024) {
229
- human_readable_speed = StringPrintf("%dB/s", bytes_per_second);
230
- } else if (bytes_per_second < 1024 * 1024) {
231
- human_readable_speed = StringPrintf(
232
- "%.1fkB/s", bytes_per_second / 1024.0f);
233
- } else if (bytes_per_second < 1024 * 1024 * 1024) {
234
- human_readable_speed = StringPrintf(
235
- "%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
236
- } else {
237
- human_readable_speed = StringPrintf(
238
- "%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
239
- }
240
- }
241
-
242
- fprintf(stderr,
243
- #ifdef WIN32
244
- "%-18s %10I64d %10I64d %10d %s %s\n",
245
- #else
246
- "%-18s %10lld %10lld %10d %s %s\n",
247
- #endif
248
- heading.c_str(),
249
- static_cast<long long>(real_time_us * 1000 / num_iterations),
250
- static_cast<long long>(cpu_time_us * 1000 / num_iterations),
251
- num_iterations,
252
- human_readable_speed.c_str(),
253
- benchmark_label->c_str());
254
- }
255
- }
256
-
257
- #ifdef HAVE_LIBZ
258
-
259
- ZLib::ZLib()
260
- : comp_init_(false),
261
- uncomp_init_(false) {
262
- Reinit();
263
- }
264
-
265
- ZLib::~ZLib() {
266
- if (comp_init_) { deflateEnd(&comp_stream_); }
267
- if (uncomp_init_) { inflateEnd(&uncomp_stream_); }
268
- }
269
-
270
- void ZLib::Reinit() {
271
- compression_level_ = Z_DEFAULT_COMPRESSION;
272
- window_bits_ = MAX_WBITS;
273
- mem_level_ = 8; // DEF_MEM_LEVEL
274
- if (comp_init_) {
275
- deflateEnd(&comp_stream_);
276
- comp_init_ = false;
277
- }
278
- if (uncomp_init_) {
279
- inflateEnd(&uncomp_stream_);
280
- uncomp_init_ = false;
281
- }
282
- first_chunk_ = true;
283
- }
284
-
285
- void ZLib::Reset() {
286
- first_chunk_ = true;
287
- }
288
-
289
- // --------- COMPRESS MODE
290
-
291
- // Initialization method to be called if we hit an error while
292
- // compressing. On hitting an error, call this method before returning
293
- // the error.
294
- void ZLib::CompressErrorInit() {
295
- deflateEnd(&comp_stream_);
296
- comp_init_ = false;
297
- Reset();
298
- }
299
-
300
- int ZLib::DeflateInit() {
301
- return deflateInit2(&comp_stream_,
302
- compression_level_,
303
- Z_DEFLATED,
304
- window_bits_,
305
- mem_level_,
306
- Z_DEFAULT_STRATEGY);
307
- }
308
-
309
- int ZLib::CompressInit(Bytef *dest, uLongf *destLen,
310
- const Bytef *source, uLong *sourceLen) {
311
- int err;
312
-
313
- comp_stream_.next_in = (Bytef*)source;
314
- comp_stream_.avail_in = (uInt)*sourceLen;
315
- if ((uLong)comp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
316
- comp_stream_.next_out = dest;
317
- comp_stream_.avail_out = (uInt)*destLen;
318
- if ((uLong)comp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
319
-
320
- if ( !first_chunk_ ) // only need to set up stream the first time through
321
- return Z_OK;
322
-
323
- if (comp_init_) { // we've already initted it
324
- err = deflateReset(&comp_stream_);
325
- if (err != Z_OK) {
326
- LOG(WARNING) << "ERROR: Can't reset compress object; creating a new one";
327
- deflateEnd(&comp_stream_);
328
- comp_init_ = false;
329
- }
330
- }
331
- if (!comp_init_) { // first use
332
- comp_stream_.zalloc = (alloc_func)0;
333
- comp_stream_.zfree = (free_func)0;
334
- comp_stream_.opaque = (voidpf)0;
335
- err = DeflateInit();
336
- if (err != Z_OK) return err;
337
- comp_init_ = true;
338
- }
339
- return Z_OK;
340
- }
341
-
342
- // In a perfect world we'd always have the full buffer to compress
343
- // when the time came, and we could just call Compress(). Alas, we
344
- // want to do chunked compression on our webserver. In this
345
- // application, we compress the header, send it off, then compress the
346
- // results, send them off, then compress the footer. Thus we need to
347
- // use the chunked compression features of zlib.
348
- int ZLib::CompressAtMostOrAll(Bytef *dest, uLongf *destLen,
349
- const Bytef *source, uLong *sourceLen,
350
- int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
351
- int err;
352
-
353
- if ( (err=CompressInit(dest, destLen, source, sourceLen)) != Z_OK )
354
- return err;
355
-
356
- // This is used to figure out how many bytes we wrote *this chunk*
357
- int compressed_size = comp_stream_.total_out;
358
-
359
- // Some setup happens only for the first chunk we compress in a run
360
- if ( first_chunk_ ) {
361
- first_chunk_ = false;
362
- }
363
-
364
- // flush_mode is Z_FINISH for all mode, Z_SYNC_FLUSH for incremental
365
- // compression.
366
- err = deflate(&comp_stream_, flush_mode);
367
-
368
- *sourceLen = comp_stream_.avail_in;
369
-
370
- if ((err == Z_STREAM_END || err == Z_OK)
371
- && comp_stream_.avail_in == 0
372
- && comp_stream_.avail_out != 0 ) {
373
- // we processed everything ok and the output buffer was large enough.
374
- ;
375
- } else if (err == Z_STREAM_END && comp_stream_.avail_in > 0) {
376
- return Z_BUF_ERROR; // should never happen
377
- } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
378
- // an error happened
379
- CompressErrorInit();
380
- return err;
381
- } else if (comp_stream_.avail_out == 0) { // not enough space
382
- err = Z_BUF_ERROR;
383
- }
384
-
385
- assert(err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR);
386
- if (err == Z_STREAM_END)
387
- err = Z_OK;
388
-
389
- // update the crc and other metadata
390
- compressed_size = comp_stream_.total_out - compressed_size; // delta
391
- *destLen = compressed_size;
392
-
393
- return err;
394
- }
395
-
396
- int ZLib::CompressChunkOrAll(Bytef *dest, uLongf *destLen,
397
- const Bytef *source, uLong sourceLen,
398
- int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
399
- const int ret =
400
- CompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
401
- if (ret == Z_BUF_ERROR)
402
- CompressErrorInit();
403
- return ret;
404
- }
405
-
406
- // This routine only initializes the compression stream once. Thereafter, it
407
- // just does a deflateReset on the stream, which should be faster.
408
- int ZLib::Compress(Bytef *dest, uLongf *destLen,
409
- const Bytef *source, uLong sourceLen) {
410
- int err;
411
- if ( (err=CompressChunkOrAll(dest, destLen, source, sourceLen,
412
- Z_FINISH)) != Z_OK )
413
- return err;
414
- Reset(); // reset for next call to Compress
415
-
416
- return Z_OK;
417
- }
418
-
419
-
420
- // --------- UNCOMPRESS MODE
421
-
422
- int ZLib::InflateInit() {
423
- return inflateInit2(&uncomp_stream_, MAX_WBITS);
424
- }
425
-
426
- // Initialization method to be called if we hit an error while
427
- // uncompressing. On hitting an error, call this method before
428
- // returning the error.
429
- void ZLib::UncompressErrorInit() {
430
- inflateEnd(&uncomp_stream_);
431
- uncomp_init_ = false;
432
- Reset();
433
- }
434
-
435
- int ZLib::UncompressInit(Bytef *dest, uLongf *destLen,
436
- const Bytef *source, uLong *sourceLen) {
437
- int err;
438
-
439
- uncomp_stream_.next_in = (Bytef*)source;
440
- uncomp_stream_.avail_in = (uInt)*sourceLen;
441
- // Check for source > 64K on 16-bit machine:
442
- if ((uLong)uncomp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
443
-
444
- uncomp_stream_.next_out = dest;
445
- uncomp_stream_.avail_out = (uInt)*destLen;
446
- if ((uLong)uncomp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
447
-
448
- if ( !first_chunk_ ) // only need to set up stream the first time through
449
- return Z_OK;
450
-
451
- if (uncomp_init_) { // we've already initted it
452
- err = inflateReset(&uncomp_stream_);
453
- if (err != Z_OK) {
454
- LOG(WARNING)
455
- << "ERROR: Can't reset uncompress object; creating a new one";
456
- UncompressErrorInit();
457
- }
458
- }
459
- if (!uncomp_init_) {
460
- uncomp_stream_.zalloc = (alloc_func)0;
461
- uncomp_stream_.zfree = (free_func)0;
462
- uncomp_stream_.opaque = (voidpf)0;
463
- err = InflateInit();
464
- if (err != Z_OK) return err;
465
- uncomp_init_ = true;
466
- }
467
- return Z_OK;
468
- }
469
-
470
- // If you compressed your data a chunk at a time, with CompressChunk,
471
- // you can uncompress it a chunk at a time with UncompressChunk.
472
- // Only difference bewteen chunked and unchunked uncompression
473
- // is the flush mode we use: Z_SYNC_FLUSH (chunked) or Z_FINISH (unchunked).
474
- int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
475
- const Bytef *source, uLong *sourceLen,
476
- int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
477
- int err = Z_OK;
478
-
479
- if ( (err=UncompressInit(dest, destLen, source, sourceLen)) != Z_OK ) {
480
- LOG(WARNING) << "UncompressInit: Error: " << err << " SourceLen: "
481
- << *sourceLen;
482
- return err;
483
- }
484
-
485
- // This is used to figure out how many output bytes we wrote *this chunk*:
486
- const uLong old_total_out = uncomp_stream_.total_out;
487
-
488
- // This is used to figure out how many input bytes we read *this chunk*:
489
- const uLong old_total_in = uncomp_stream_.total_in;
490
-
491
- // Some setup happens only for the first chunk we compress in a run
492
- if ( first_chunk_ ) {
493
- first_chunk_ = false; // so we don't do this again
494
-
495
- // For the first chunk *only* (to avoid infinite troubles), we let
496
- // there be no actual data to uncompress. This sometimes triggers
497
- // when the input is only the gzip header, say.
498
- if ( *sourceLen == 0 ) {
499
- *destLen = 0;
500
- return Z_OK;
501
- }
502
- }
503
-
504
- // We'll uncompress as much as we can. If we end OK great, otherwise
505
- // if we get an error that seems to be the gzip footer, we store the
506
- // gzip footer and return OK, otherwise we return the error.
507
-
508
- // flush_mode is Z_SYNC_FLUSH for chunked mode, Z_FINISH for all mode.
509
- err = inflate(&uncomp_stream_, flush_mode);
510
-
511
- // Figure out how many bytes of the input zlib slurped up:
512
- const uLong bytes_read = uncomp_stream_.total_in - old_total_in;
513
- CHECK_LE(source + bytes_read, source + *sourceLen);
514
- *sourceLen = uncomp_stream_.avail_in;
515
-
516
- if ((err == Z_STREAM_END || err == Z_OK) // everything went ok
517
- && uncomp_stream_.avail_in == 0) { // and we read it all
518
- ;
519
- } else if (err == Z_STREAM_END && uncomp_stream_.avail_in > 0) {
520
- LOG(WARNING)
521
- << "UncompressChunkOrAll: Received some extra data, bytes total: "
522
- << uncomp_stream_.avail_in << " bytes: "
523
- << string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
524
- min(int(uncomp_stream_.avail_in), 20));
525
- UncompressErrorInit();
526
- return Z_DATA_ERROR; // what's the extra data for?
527
- } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
528
- // an error happened
529
- LOG(WARNING) << "UncompressChunkOrAll: Error: " << err
530
- << " avail_out: " << uncomp_stream_.avail_out;
531
- UncompressErrorInit();
532
- return err;
533
- } else if (uncomp_stream_.avail_out == 0) {
534
- err = Z_BUF_ERROR;
535
- }
536
-
537
- assert(err == Z_OK || err == Z_BUF_ERROR || err == Z_STREAM_END);
538
- if (err == Z_STREAM_END)
539
- err = Z_OK;
540
-
541
- *destLen = uncomp_stream_.total_out - old_total_out; // size for this call
542
-
543
- return err;
544
- }
545
-
546
- int ZLib::UncompressChunkOrAll(Bytef *dest, uLongf *destLen,
547
- const Bytef *source, uLong sourceLen,
548
- int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
549
- const int ret =
550
- UncompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
551
- if (ret == Z_BUF_ERROR)
552
- UncompressErrorInit();
553
- return ret;
554
- }
555
-
556
- int ZLib::UncompressAtMost(Bytef *dest, uLongf *destLen,
557
- const Bytef *source, uLong *sourceLen) {
558
- return UncompressAtMostOrAll(dest, destLen, source, sourceLen, Z_SYNC_FLUSH);
559
- }
560
-
561
- // We make sure we've uncompressed everything, that is, the current
562
- // uncompress stream is at a compressed-buffer-EOF boundary. In gzip
563
- // mode, we also check the gzip footer to make sure we pass the gzip
564
- // consistency checks. We RETURN true iff both types of checks pass.
565
- bool ZLib::UncompressChunkDone() {
566
- assert(!first_chunk_ && uncomp_init_);
567
- // Make sure we're at the end-of-compressed-data point. This means
568
- // if we call inflate with Z_FINISH we won't consume any input or
569
- // write any output
570
- Bytef dummyin, dummyout;
571
- uLongf dummylen = 0;
572
- if ( UncompressChunkOrAll(&dummyout, &dummylen, &dummyin, 0, Z_FINISH)
573
- != Z_OK ) {
574
- return false;
575
- }
576
-
577
- // Make sure that when we exit, we can start a new round of chunks later
578
- Reset();
579
-
580
- return true;
581
- }
582
-
583
- // Uncompresses the source buffer into the destination buffer.
584
- // The destination buffer must be long enough to hold the entire
585
- // decompressed contents.
586
- //
587
- // We only initialize the uncomp_stream once. Thereafter, we use
588
- // inflateReset, which should be faster.
589
- //
590
- // Returns Z_OK on success, otherwise, it returns a zlib error code.
591
- int ZLib::Uncompress(Bytef *dest, uLongf *destLen,
592
- const Bytef *source, uLong sourceLen) {
593
- int err;
594
- if ( (err=UncompressChunkOrAll(dest, destLen, source, sourceLen,
595
- Z_FINISH)) != Z_OK ) {
596
- Reset(); // let us try to compress again
597
- return err;
598
- }
599
- if ( !UncompressChunkDone() ) // calls Reset()
600
- return Z_DATA_ERROR;
601
- return Z_OK; // stream_end is ok
602
- }
603
-
604
- #endif // HAVE_LIBZ
605
-
606
- } // namespace snappy