deflate-ruby 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CLAUDE.md +138 -0
- data/LICENSE.txt +21 -0
- data/README.md +117 -0
- data/ext/deflate_ruby/deflate_ruby.c +301 -0
- data/ext/deflate_ruby/extconf.rb +34 -0
- data/ext/deflate_ruby/libdeflate/CMakeLists.txt +270 -0
- data/ext/deflate_ruby/libdeflate/COPYING +22 -0
- data/ext/deflate_ruby/libdeflate/NEWS.md +494 -0
- data/ext/deflate_ruby/libdeflate/README.md +228 -0
- data/ext/deflate_ruby/libdeflate/common_defs.h +747 -0
- data/ext/deflate_ruby/libdeflate/lib/adler32.c +162 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/adler32_impl.h +358 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/cpu_features.c +230 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/cpu_features.h +214 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/crc32_impl.h +600 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/crc32_pmull_helpers.h +156 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/crc32_pmull_wide.h +226 -0
- data/ext/deflate_ruby/libdeflate/lib/arm/matchfinder_impl.h +78 -0
- data/ext/deflate_ruby/libdeflate/lib/bt_matchfinder.h +342 -0
- data/ext/deflate_ruby/libdeflate/lib/cpu_features_common.h +93 -0
- data/ext/deflate_ruby/libdeflate/lib/crc32.c +262 -0
- data/ext/deflate_ruby/libdeflate/lib/crc32_multipliers.h +377 -0
- data/ext/deflate_ruby/libdeflate/lib/crc32_tables.h +587 -0
- data/ext/deflate_ruby/libdeflate/lib/decompress_template.h +777 -0
- data/ext/deflate_ruby/libdeflate/lib/deflate_compress.c +4129 -0
- data/ext/deflate_ruby/libdeflate/lib/deflate_compress.h +15 -0
- data/ext/deflate_ruby/libdeflate/lib/deflate_constants.h +56 -0
- data/ext/deflate_ruby/libdeflate/lib/deflate_decompress.c +1208 -0
- data/ext/deflate_ruby/libdeflate/lib/gzip_compress.c +90 -0
- data/ext/deflate_ruby/libdeflate/lib/gzip_constants.h +45 -0
- data/ext/deflate_ruby/libdeflate/lib/gzip_decompress.c +144 -0
- data/ext/deflate_ruby/libdeflate/lib/hc_matchfinder.h +401 -0
- data/ext/deflate_ruby/libdeflate/lib/ht_matchfinder.h +234 -0
- data/ext/deflate_ruby/libdeflate/lib/lib_common.h +106 -0
- data/ext/deflate_ruby/libdeflate/lib/matchfinder_common.h +224 -0
- data/ext/deflate_ruby/libdeflate/lib/riscv/matchfinder_impl.h +97 -0
- data/ext/deflate_ruby/libdeflate/lib/utils.c +141 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/adler32_impl.h +134 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/adler32_template.h +518 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/cpu_features.c +183 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/cpu_features.h +169 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/crc32_impl.h +160 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/crc32_pclmul_template.h +495 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/decompress_impl.h +57 -0
- data/ext/deflate_ruby/libdeflate/lib/x86/matchfinder_impl.h +122 -0
- data/ext/deflate_ruby/libdeflate/lib/zlib_compress.c +82 -0
- data/ext/deflate_ruby/libdeflate/lib/zlib_constants.h +21 -0
- data/ext/deflate_ruby/libdeflate/lib/zlib_decompress.c +104 -0
- data/ext/deflate_ruby/libdeflate/libdeflate-config.cmake.in +3 -0
- data/ext/deflate_ruby/libdeflate/libdeflate.h +411 -0
- data/ext/deflate_ruby/libdeflate/libdeflate.pc.in +18 -0
- data/ext/deflate_ruby/libdeflate/programs/CMakeLists.txt +105 -0
- data/ext/deflate_ruby/libdeflate/programs/benchmark.c +696 -0
- data/ext/deflate_ruby/libdeflate/programs/checksum.c +218 -0
- data/ext/deflate_ruby/libdeflate/programs/config.h.in +19 -0
- data/ext/deflate_ruby/libdeflate/programs/gzip.c +688 -0
- data/ext/deflate_ruby/libdeflate/programs/prog_util.c +521 -0
- data/ext/deflate_ruby/libdeflate/programs/prog_util.h +225 -0
- data/ext/deflate_ruby/libdeflate/programs/test_checksums.c +200 -0
- data/ext/deflate_ruby/libdeflate/programs/test_custom_malloc.c +155 -0
- data/ext/deflate_ruby/libdeflate/programs/test_incomplete_codes.c +385 -0
- data/ext/deflate_ruby/libdeflate/programs/test_invalid_streams.c +130 -0
- data/ext/deflate_ruby/libdeflate/programs/test_litrunlen_overflow.c +72 -0
- data/ext/deflate_ruby/libdeflate/programs/test_overread.c +95 -0
- data/ext/deflate_ruby/libdeflate/programs/test_slow_decompression.c +472 -0
- data/ext/deflate_ruby/libdeflate/programs/test_trailing_bytes.c +151 -0
- data/ext/deflate_ruby/libdeflate/programs/test_util.c +237 -0
- data/ext/deflate_ruby/libdeflate/programs/test_util.h +61 -0
- data/ext/deflate_ruby/libdeflate/programs/tgetopt.c +118 -0
- data/ext/deflate_ruby/libdeflate/scripts/android_build.sh +118 -0
- data/ext/deflate_ruby/libdeflate/scripts/android_tests.sh +69 -0
- data/ext/deflate_ruby/libdeflate/scripts/benchmark.sh +10 -0
- data/ext/deflate_ruby/libdeflate/scripts/checksum.sh +10 -0
- data/ext/deflate_ruby/libdeflate/scripts/checksum_benchmarks.sh +253 -0
- data/ext/deflate_ruby/libdeflate/scripts/cmake-helper.sh +17 -0
- data/ext/deflate_ruby/libdeflate/scripts/deflate_benchmarks.sh +119 -0
- data/ext/deflate_ruby/libdeflate/scripts/exec_tests.sh +38 -0
- data/ext/deflate_ruby/libdeflate/scripts/gen-release-archives.sh +37 -0
- data/ext/deflate_ruby/libdeflate/scripts/gen_bitreverse_tab.py +19 -0
- data/ext/deflate_ruby/libdeflate/scripts/gen_crc32_multipliers.c +199 -0
- data/ext/deflate_ruby/libdeflate/scripts/gen_crc32_tables.c +105 -0
- data/ext/deflate_ruby/libdeflate/scripts/gen_default_litlen_costs.py +44 -0
- data/ext/deflate_ruby/libdeflate/scripts/gen_offset_slot_map.py +29 -0
- data/ext/deflate_ruby/libdeflate/scripts/gzip_tests.sh +523 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/deflate_compress/corpus/0 +0 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/deflate_compress/fuzz.c +95 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/deflate_decompress/corpus/0 +3 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/deflate_decompress/fuzz.c +62 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/fuzz.sh +108 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/gzip_decompress/corpus/0 +0 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/gzip_decompress/fuzz.c +19 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/zlib_decompress/corpus/0 +3 -0
- data/ext/deflate_ruby/libdeflate/scripts/libFuzzer/zlib_decompress/fuzz.c +19 -0
- data/ext/deflate_ruby/libdeflate/scripts/run_tests.sh +416 -0
- data/ext/deflate_ruby/libdeflate/scripts/toolchain-i686-w64-mingw32.cmake +8 -0
- data/ext/deflate_ruby/libdeflate/scripts/toolchain-x86_64-w64-mingw32.cmake +8 -0
- data/lib/deflate_ruby/version.rb +5 -0
- data/lib/deflate_ruby.rb +71 -0
- metadata +191 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* gen_crc32_tables.c - a program for CRC-32 table generation
|
|
3
|
+
*
|
|
4
|
+
* Copyright 2016 Eric Biggers
|
|
5
|
+
*
|
|
6
|
+
* Permission is hereby granted, free of charge, to any person
|
|
7
|
+
* obtaining a copy of this software and associated documentation
|
|
8
|
+
* files (the "Software"), to deal in the Software without
|
|
9
|
+
* restriction, including without limitation the rights to use,
|
|
10
|
+
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
+
* copies of the Software, and to permit persons to whom the
|
|
12
|
+
* Software is furnished to do so, subject to the following
|
|
13
|
+
* conditions:
|
|
14
|
+
*
|
|
15
|
+
* The above copyright notice and this permission notice shall be
|
|
16
|
+
* included in all copies or substantial portions of the Software.
|
|
17
|
+
*
|
|
18
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
19
|
+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|
20
|
+
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
21
|
+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
22
|
+
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
23
|
+
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
24
|
+
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
25
|
+
* OTHER DEALINGS IN THE SOFTWARE.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
#include <stdio.h>
|
|
29
|
+
|
|
30
|
+
#include "../common_defs.h"
|
|
31
|
+
|
|
32
|
+
#define CRCPOLY 0xEDB88320 /* G(x) without x^32 term */
|
|
33
|
+
|
|
34
|
+
static u32
|
|
35
|
+
crc32_update_bit(u32 remainder, u8 next_bit)
|
|
36
|
+
{
|
|
37
|
+
return (remainder >> 1) ^ (((remainder ^ next_bit) & 1) ? CRCPOLY : 0);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
static u32
|
|
41
|
+
crc32_update_byte(u32 remainder, u8 next_byte)
|
|
42
|
+
{
|
|
43
|
+
for (int j = 0; j < 8; j++, next_byte >>= 1)
|
|
44
|
+
remainder = crc32_update_bit(remainder, next_byte & 1);
|
|
45
|
+
return remainder;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
static void
|
|
49
|
+
print_256_entries(const u32 *entries)
|
|
50
|
+
{
|
|
51
|
+
for (size_t i = 0; i < 256 / 4; i++) {
|
|
52
|
+
printf("\t");
|
|
53
|
+
for (size_t j = 0; j < 4; j++) {
|
|
54
|
+
printf("0x%08x,", entries[i * 4 + j]);
|
|
55
|
+
if (j != 3)
|
|
56
|
+
printf(" ");
|
|
57
|
+
}
|
|
58
|
+
printf("\n");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
int
|
|
63
|
+
main(void)
|
|
64
|
+
{
|
|
65
|
+
u32 crc32_table[0x800];
|
|
66
|
+
|
|
67
|
+
/* crc32_table[i] for 0 <= i < 0x100 is the CRC-32 of byte i. */
|
|
68
|
+
for (int i = 0; i < 0x100; i++)
|
|
69
|
+
crc32_table[i] = crc32_update_byte(0, i);
|
|
70
|
+
|
|
71
|
+
/*
|
|
72
|
+
* crc32_table[i] for 0x100 <= i < 0x800 is the CRC-32 of byte i % 0x100
|
|
73
|
+
* followed by i / 0x100 zero bytes.
|
|
74
|
+
*/
|
|
75
|
+
for (int i = 0x100; i < 0x800; i++)
|
|
76
|
+
crc32_table[i] = crc32_update_byte(crc32_table[i - 0x100], 0);
|
|
77
|
+
|
|
78
|
+
printf("/*\n");
|
|
79
|
+
printf(" * crc32_tables.h - data tables for CRC-32 computation\n");
|
|
80
|
+
printf(" *\n");
|
|
81
|
+
printf(" * THIS FILE WAS GENERATED BY gen_crc32_tables.c. DO NOT EDIT.\n");
|
|
82
|
+
printf(" */\n");
|
|
83
|
+
printf("\n");
|
|
84
|
+
/*
|
|
85
|
+
* Although crc32_slice1_table is the same as the first 256 entries of
|
|
86
|
+
* crc32_slice8_table, we output these tables separately because any
|
|
87
|
+
* combo of (slice1, slice8, slice1 && slice8, nothing) might be needed,
|
|
88
|
+
* and it's simplest to let the compiler optimize out any unused tables.
|
|
89
|
+
*/
|
|
90
|
+
printf("static const u32 crc32_slice1_table[] MAYBE_UNUSED = {\n");
|
|
91
|
+
print_256_entries(&crc32_table[0x000]);
|
|
92
|
+
printf("};\n");
|
|
93
|
+
printf("\n");
|
|
94
|
+
printf("static const u32 crc32_slice8_table[] MAYBE_UNUSED = {\n");
|
|
95
|
+
print_256_entries(&crc32_table[0x000]);
|
|
96
|
+
print_256_entries(&crc32_table[0x100]);
|
|
97
|
+
print_256_entries(&crc32_table[0x200]);
|
|
98
|
+
print_256_entries(&crc32_table[0x300]);
|
|
99
|
+
print_256_entries(&crc32_table[0x400]);
|
|
100
|
+
print_256_entries(&crc32_table[0x500]);
|
|
101
|
+
print_256_entries(&crc32_table[0x600]);
|
|
102
|
+
print_256_entries(&crc32_table[0x700]);
|
|
103
|
+
printf("};\n");
|
|
104
|
+
return 0;
|
|
105
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
# This script computes the default litlen symbol costs for the near-optimal
|
|
4
|
+
# parser.
|
|
5
|
+
|
|
6
|
+
from math import log2
|
|
7
|
+
|
|
8
|
+
BIT_COST = 16 # Must match BIT_COST in deflate_compress.c
|
|
9
|
+
NUM_LEN_SLOTS = 29
|
|
10
|
+
|
|
11
|
+
print("""static const struct {
|
|
12
|
+
u8 used_lits_to_lit_cost[257];
|
|
13
|
+
u8 len_sym_cost;
|
|
14
|
+
} default_litlen_costs[] = {""")
|
|
15
|
+
MATCH_PROBS = [0.25, 0.50, 0.75]
|
|
16
|
+
for i, match_prob in enumerate(MATCH_PROBS):
|
|
17
|
+
len_prob = match_prob / NUM_LEN_SLOTS
|
|
18
|
+
len_sym_cost = int(-log2(len_prob) * BIT_COST)
|
|
19
|
+
if i == 0:
|
|
20
|
+
print('\t{', end='')
|
|
21
|
+
print(f' /* match_prob = {match_prob} */')
|
|
22
|
+
print('\t\t.used_lits_to_lit_cost = {')
|
|
23
|
+
|
|
24
|
+
j = 0
|
|
25
|
+
for num_used_literals in range(0, 257):
|
|
26
|
+
if num_used_literals == 0:
|
|
27
|
+
num_used_literals = 1
|
|
28
|
+
lit_prob = (1 - match_prob) / num_used_literals
|
|
29
|
+
lit_cost = int(-log2(lit_prob) * BIT_COST)
|
|
30
|
+
if j == 0:
|
|
31
|
+
print('\t\t\t', end='')
|
|
32
|
+
if j == 7 or num_used_literals == 256:
|
|
33
|
+
print(f'{lit_cost},')
|
|
34
|
+
j = 0
|
|
35
|
+
else:
|
|
36
|
+
print(f'{lit_cost}, ', end='')
|
|
37
|
+
j += 1
|
|
38
|
+
print('\t\t},')
|
|
39
|
+
print(f'\t\t.len_sym_cost = {len_sym_cost},')
|
|
40
|
+
if i < len(MATCH_PROBS) - 1:
|
|
41
|
+
print('\t}, {', end='')
|
|
42
|
+
else:
|
|
43
|
+
print('\t},')
|
|
44
|
+
print('};')
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
# This script generates the deflate_offset_slot[] array, which maps
|
|
4
|
+
# 'offset - 1 => offset_slot' for offset <= 256.
|
|
5
|
+
|
|
6
|
+
DEFLATE_OFFSET_SLOT_BASE = [
|
|
7
|
+
1 , 2 , 3 , 4 , 5 , 7 , 9 , 13 ,
|
|
8
|
+
17 , 25 , 33 , 49 , 65 , 97 , 129 , 193 ,
|
|
9
|
+
257 , 385 , 513 , 769 , 1025 , 1537 , 2049 , 3073 ,
|
|
10
|
+
4097 , 6145 , 8193 , 12289 , 16385 , 24577 ,
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
offset_slot_map = [0] * 256
|
|
14
|
+
offset_slot = -1
|
|
15
|
+
for offset in range(1, len(offset_slot_map) + 1):
|
|
16
|
+
if offset >= DEFLATE_OFFSET_SLOT_BASE[offset_slot + 1]:
|
|
17
|
+
offset_slot += 1
|
|
18
|
+
offset_slot_map[offset - 1] = offset_slot
|
|
19
|
+
|
|
20
|
+
print(f'static const u8 deflate_offset_slot[{len(offset_slot_map)}] = {{')
|
|
21
|
+
for i in range(0, len(offset_slot_map), 16):
|
|
22
|
+
print('\t', end='')
|
|
23
|
+
for j, v in enumerate(offset_slot_map[i:i+16]):
|
|
24
|
+
print(f'{v},', end='')
|
|
25
|
+
if j == 15:
|
|
26
|
+
print('')
|
|
27
|
+
else:
|
|
28
|
+
print(' ', end='')
|
|
29
|
+
print('};')
|
|
@@ -0,0 +1,523 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
#
|
|
3
|
+
# Test script for libdeflate's gzip and gunzip programs.
|
|
4
|
+
#
|
|
5
|
+
# To run, you must set GZIP and GUNZIP in the environment to the absolute paths
|
|
6
|
+
# to the gzip and gunzip programs to test. All tests should pass regardless of
|
|
7
|
+
# whether the GNU versions or the libdeflate versions, or a combination, of
|
|
8
|
+
# these programs are used.
|
|
9
|
+
#
|
|
10
|
+
# The environmental variable TESTDATA must also be set to a file containing
|
|
11
|
+
# test data.
|
|
12
|
+
#
|
|
13
|
+
|
|
14
|
+
set -eu -o pipefail
|
|
15
|
+
|
|
16
|
+
export -n GZIP GUNZIP TESTDATA
|
|
17
|
+
|
|
18
|
+
ORIG_PWD=$PWD
|
|
19
|
+
TMPDIR="$(mktemp -d)"
|
|
20
|
+
CURRENT_TEST=
|
|
21
|
+
|
|
22
|
+
BSD_STAT=false
|
|
23
|
+
if ! stat --version 2>&1 | grep -q coreutils; then
|
|
24
|
+
BSD_STAT=true
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
cleanup() {
|
|
28
|
+
if [ -n "$CURRENT_TEST" ]; then
|
|
29
|
+
echo "TEST FAILED: \"$CURRENT_TEST\""
|
|
30
|
+
fi
|
|
31
|
+
rm -rf -- "$TMPDIR"
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
trap cleanup EXIT
|
|
35
|
+
|
|
36
|
+
begin_test() {
|
|
37
|
+
CURRENT_TEST="$1"
|
|
38
|
+
rm -rf -- "${TMPDIR:?}"/*
|
|
39
|
+
cd "$ORIG_PWD"
|
|
40
|
+
cp "$TESTDATA" "$TMPDIR/file"
|
|
41
|
+
chmod +w "$TMPDIR/file"
|
|
42
|
+
cd "$TMPDIR"
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
gzip() {
|
|
46
|
+
$GZIP "$@"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
gunzip() {
|
|
50
|
+
$GUNZIP "$@"
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
get_filesize() {
|
|
54
|
+
local file=$1
|
|
55
|
+
|
|
56
|
+
if $BSD_STAT; then
|
|
57
|
+
stat -f %z "$file"
|
|
58
|
+
else
|
|
59
|
+
stat -c %s "$file"
|
|
60
|
+
fi
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
get_linkcount() {
|
|
64
|
+
local file=$1
|
|
65
|
+
|
|
66
|
+
if $BSD_STAT; then
|
|
67
|
+
stat -f %l "$file"
|
|
68
|
+
else
|
|
69
|
+
stat -c %h "$file"
|
|
70
|
+
fi
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
get_modeandtimestamps() {
|
|
74
|
+
local file=$1
|
|
75
|
+
|
|
76
|
+
if $BSD_STAT; then
|
|
77
|
+
stat -f "%p;%a;%m" "$file"
|
|
78
|
+
elif [ "$(uname -m)" = s390x ]; then
|
|
79
|
+
# Use seconds precision instead of nanoseconds.
|
|
80
|
+
# TODO: why is this needed? QEMU user mode emulation bug?
|
|
81
|
+
stat -c "%a;%X;%Y" "$file"
|
|
82
|
+
else
|
|
83
|
+
stat -c "%a;%x;%y" "$file"
|
|
84
|
+
fi
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
assert_status() {
|
|
88
|
+
local expected_status="$1"
|
|
89
|
+
local expected_msg="$2"
|
|
90
|
+
shift 2
|
|
91
|
+
(
|
|
92
|
+
set +e
|
|
93
|
+
{ eval "$*" > /dev/null; } 2>&1
|
|
94
|
+
local actual_status=$?
|
|
95
|
+
if [ "$actual_status" != "$expected_status" ]; then
|
|
96
|
+
echo 1>&2 "Command '$*' exited with status" \
|
|
97
|
+
"$actual_status but expected status" \
|
|
98
|
+
"$expected_status"
|
|
99
|
+
exit 1
|
|
100
|
+
fi
|
|
101
|
+
exit 0
|
|
102
|
+
) > command_output
|
|
103
|
+
if ! grep -E -q "$expected_msg" command_output; then
|
|
104
|
+
echo 1>&2 "Expected output of command '$*' to match regex" \
|
|
105
|
+
"'$expected_msg'"
|
|
106
|
+
echo 1>&2 "Actual output was:"
|
|
107
|
+
echo 1>&2 "---------------------------------------------------"
|
|
108
|
+
cat 1>&2 command_output
|
|
109
|
+
echo 1>&2 "---------------------------------------------------"
|
|
110
|
+
return 1
|
|
111
|
+
fi
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
assert_error() {
|
|
115
|
+
assert_status 1 "$@"
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
assert_warning() {
|
|
119
|
+
assert_status 2 "$@"
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
assert_skipped() {
|
|
123
|
+
assert_warning '\<(ignored|skipping|unchanged)\>' "$@"
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
assert_equals() {
|
|
127
|
+
local expected="$1"
|
|
128
|
+
local actual="$2"
|
|
129
|
+
|
|
130
|
+
if [ "$expected" != "$actual" ]; then
|
|
131
|
+
echo 1>&2 "Expected '$expected', but got '$actual'"
|
|
132
|
+
return 1
|
|
133
|
+
fi
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
begin_test 'Basic compression and decompression works'
|
|
138
|
+
cp file orig
|
|
139
|
+
gzip file
|
|
140
|
+
[ ! -e file ] && [ -e file.gz ]
|
|
141
|
+
gunzip file.gz
|
|
142
|
+
[ -e file ] && [ ! -e file.gz ]
|
|
143
|
+
cmp file orig
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
begin_test 'gzip -d is gunzip'
|
|
147
|
+
cp file orig
|
|
148
|
+
gzip file
|
|
149
|
+
gzip -d file.gz
|
|
150
|
+
cmp file orig
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
begin_test '-k (keep original file) works'
|
|
154
|
+
cp file orig
|
|
155
|
+
gzip -k file
|
|
156
|
+
cmp file orig
|
|
157
|
+
rm file
|
|
158
|
+
cp file.gz orig.gz
|
|
159
|
+
gunzip -k file.gz
|
|
160
|
+
cmp file.gz orig.gz
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
begin_test '-c (write to stdout) works'
|
|
164
|
+
cp file orig
|
|
165
|
+
gzip -k file
|
|
166
|
+
gzip -c file > 2.gz
|
|
167
|
+
cmp file orig
|
|
168
|
+
cmp file.gz 2.gz
|
|
169
|
+
gunzip -c 2.gz > file
|
|
170
|
+
cmp file.gz 2.gz
|
|
171
|
+
cmp file orig
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
# Note: in some of the commands below, we intentionally use 'cat file | gzip'
|
|
175
|
+
# rather than 'gzip < file', in order to test the use of a pipe. This produces
|
|
176
|
+
# a shellcheck warning about 'cat' being unnecessary. Suppress that warning by
|
|
177
|
+
# using { cat file; true; }.
|
|
178
|
+
begin_test 'Reading from stdin works'
|
|
179
|
+
gzip < file > 1.gz
|
|
180
|
+
gzip - < file > 2.gz
|
|
181
|
+
{ cat file; true; } | gzip > 3.gz
|
|
182
|
+
{ cat file; true; } | gzip - > 4.gz
|
|
183
|
+
cmp file <(gunzip < 1.gz)
|
|
184
|
+
cmp file <(gunzip - < 2.gz)
|
|
185
|
+
cmp file <({ cat 3.gz; true; } | gunzip)
|
|
186
|
+
cmp file <({ cat 4.gz; true; } | gunzip -)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
begin_test '-n option is accepted'
|
|
190
|
+
gzip -n file
|
|
191
|
+
gunzip -n file.gz
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
begin_test 'can specify multiple options'
|
|
195
|
+
gzip -fk1 file
|
|
196
|
+
cmp <(gzip -c -1 file) file.gz
|
|
197
|
+
gunzip -kfd file.gz
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
begin_test 'Compression levels'
|
|
201
|
+
if [ "$GZIP" = /bin/gzip ] || [ "$GZIP" = /usr/bin/gzip ]; then
|
|
202
|
+
assert_error '\<invalid option\>' gzip -10
|
|
203
|
+
max_level=9
|
|
204
|
+
else
|
|
205
|
+
for level in 13 99999 1a; do
|
|
206
|
+
assert_error '\<Invalid compression level\>' gzip -$level
|
|
207
|
+
done
|
|
208
|
+
max_level=12
|
|
209
|
+
fi
|
|
210
|
+
for level in $(seq 1 $max_level); do
|
|
211
|
+
gzip -c "-$level" file > "file$level"
|
|
212
|
+
cmp file <(gunzip -c "file$level")
|
|
213
|
+
done
|
|
214
|
+
rm file command_output
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
begin_test 'Overwriting output file requires -f'
|
|
218
|
+
cp file orig
|
|
219
|
+
echo -n > file.gz
|
|
220
|
+
gzip -c file > 2.gz
|
|
221
|
+
assert_warning 'already exists' gzip file </dev/null
|
|
222
|
+
cmp file.gz /dev/null
|
|
223
|
+
gzip -f file
|
|
224
|
+
cmp 2.gz file.gz
|
|
225
|
+
echo -n > file
|
|
226
|
+
assert_warning 'already exists' gunzip file.gz </dev/null
|
|
227
|
+
gunzip -f file.gz
|
|
228
|
+
cmp file orig
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
begin_test 'Nonexistent input file fails, even with -f'
|
|
232
|
+
for prog in 'gzip' 'gzip -f' 'gunzip' 'gunzip -f'; do
|
|
233
|
+
assert_error 'No such file or directory' "$prog" NONEXISTENT
|
|
234
|
+
done
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
begin_test 'Compressing already-suffixed file requires -f or -c'
|
|
238
|
+
gzip file
|
|
239
|
+
gzip -c file.gz > c.gz
|
|
240
|
+
gzip file.gz 2>&1 >/dev/null | grep -q 'already has .gz suffix'
|
|
241
|
+
[ -e file.gz ] && [ ! -e file.gz.gz ]
|
|
242
|
+
gzip -f file.gz
|
|
243
|
+
[ ! -e file.gz ] && [ -e file.gz.gz ]
|
|
244
|
+
cmp file.gz.gz c.gz
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
begin_test 'gunzip -f -c passes through non-gzip data'
|
|
248
|
+
echo hello > file
|
|
249
|
+
cp file orig
|
|
250
|
+
gunzip -f -c file > foo
|
|
251
|
+
cmp file foo
|
|
252
|
+
gzip file
|
|
253
|
+
gunzip -f -c file.gz > foo
|
|
254
|
+
cmp foo orig
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
begin_test 'gunzip -f (without -c) does *not* pass through non-gzip data'
|
|
258
|
+
echo hello > file.gz
|
|
259
|
+
assert_error '\<not in gzip format\>' gunzip -f file.gz
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
begin_test 'Decompressing unsuffixed file only works with -c'
|
|
263
|
+
gzip file && mv file.gz file
|
|
264
|
+
assert_skipped gunzip file
|
|
265
|
+
assert_skipped gunzip -f file
|
|
266
|
+
gunzip -c file > orig
|
|
267
|
+
mv file file.gz && gunzip file.gz && cmp file orig
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
begin_test '... unless there is a corresponding suffixed file'
|
|
271
|
+
cp file orig
|
|
272
|
+
gzip file
|
|
273
|
+
[ ! -e file ] && [ -e file.gz ]
|
|
274
|
+
gunzip -c file > tmp
|
|
275
|
+
cmp tmp orig
|
|
276
|
+
rm tmp
|
|
277
|
+
ln -s NONEXISTENT file
|
|
278
|
+
gunzip -c file > tmp
|
|
279
|
+
cmp tmp orig
|
|
280
|
+
rm tmp file
|
|
281
|
+
gunzip file
|
|
282
|
+
[ -e file ] && [ ! -e file.gz ]
|
|
283
|
+
cmp file orig
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
begin_test 'Directory is skipped, even with -f'
|
|
287
|
+
mkdir dir
|
|
288
|
+
mkdir dir.gz
|
|
289
|
+
for opt in '' '-f' '-c'; do
|
|
290
|
+
assert_skipped gzip $opt dir
|
|
291
|
+
done
|
|
292
|
+
#assert_skipped gzip dir.gz # XXX: GNU gzip warns, libdeflate gzip no-ops
|
|
293
|
+
for opt in '' '-f' '-c'; do
|
|
294
|
+
for name in dir dir.gz; do
|
|
295
|
+
assert_skipped gunzip $opt $name
|
|
296
|
+
done
|
|
297
|
+
done
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
begin_test '(gzip) symlink is rejected without -f or -c'
|
|
301
|
+
ln -s file symlink1
|
|
302
|
+
ln -s file symlink2
|
|
303
|
+
assert_error 'Too many levels of symbolic links' gzip symlink1
|
|
304
|
+
[ -e file ] && [ -e symlink1 ] && [ ! -e symlink1.gz ]
|
|
305
|
+
gzip -f symlink1
|
|
306
|
+
[ -e file ] && [ ! -e symlink1 ] && [ -e symlink1.gz ]
|
|
307
|
+
gzip -c symlink2 > /dev/null
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
begin_test '(gunzip) symlink is rejected without -f or -c'
|
|
311
|
+
gzip file
|
|
312
|
+
ln -s file.gz symlink1.gz
|
|
313
|
+
ln -s file.gz symlink2.gz
|
|
314
|
+
assert_error 'Too many levels of symbolic links' gunzip symlink1
|
|
315
|
+
[ -e file.gz ] && [ -e symlink1.gz ] && [ ! -e symlink1 ]
|
|
316
|
+
gunzip -f symlink1.gz
|
|
317
|
+
[ -e file.gz ] && [ ! -e symlink1.gz ] && [ -e symlink1 ]
|
|
318
|
+
gunzip -c symlink2.gz > /dev/null
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
begin_test 'FIFO is skipped, even with -f'
|
|
322
|
+
mkfifo foo
|
|
323
|
+
mkfifo foo.gz
|
|
324
|
+
assert_skipped gzip foo
|
|
325
|
+
assert_skipped gzip -f foo
|
|
326
|
+
#assert_skipped gzip -c foo # XXX: works with GNU gzip, not libdeflate's
|
|
327
|
+
assert_skipped gunzip foo.gz
|
|
328
|
+
assert_skipped gunzip -f foo.gz
|
|
329
|
+
#assert_skipped gunzip -c foo.gz # XXX: works with GNU gzip, not libdeflate's
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
begin_test '(gzip) overwriting symlink does not follow symlink'
|
|
333
|
+
echo a > a
|
|
334
|
+
echo b > b
|
|
335
|
+
gzip a
|
|
336
|
+
ln -s a.gz b.gz
|
|
337
|
+
gzip -f b
|
|
338
|
+
gunzip a.gz
|
|
339
|
+
cmp <(echo a) a
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
begin_test '(gunzip) overwriting symlink does not follow symlink'
|
|
343
|
+
echo a > a
|
|
344
|
+
echo b > b
|
|
345
|
+
gzip b
|
|
346
|
+
ln -s a b
|
|
347
|
+
gunzip -f b.gz
|
|
348
|
+
cmp <(echo a) a
|
|
349
|
+
cmp <(echo b) b
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
begin_test '(gzip) hard linked file skipped without -f or -c'
|
|
353
|
+
cp file orig
|
|
354
|
+
ln file link
|
|
355
|
+
assert_equals 2 "$(get_linkcount file)"
|
|
356
|
+
assert_skipped gzip file
|
|
357
|
+
gzip -c file > /dev/null
|
|
358
|
+
assert_equals 2 "$(get_linkcount file)"
|
|
359
|
+
gzip -f file
|
|
360
|
+
assert_equals 1 "$(get_linkcount link)"
|
|
361
|
+
assert_equals 1 "$(get_linkcount file.gz)"
|
|
362
|
+
cmp link orig
|
|
363
|
+
# XXX: GNU gzip skips hard linked files with -k, libdeflate's doesn't
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
begin_test '(gunzip) hard linked file skipped without -f or -c'
|
|
367
|
+
gzip file
|
|
368
|
+
ln file.gz link.gz
|
|
369
|
+
cp file.gz orig.gz
|
|
370
|
+
assert_equals 2 "$(get_linkcount file.gz)"
|
|
371
|
+
assert_skipped gunzip file.gz
|
|
372
|
+
gunzip -c file.gz > /dev/null
|
|
373
|
+
assert_equals 2 "$(get_linkcount file.gz)"
|
|
374
|
+
gunzip -f file
|
|
375
|
+
assert_equals 1 "$(get_linkcount link.gz)"
|
|
376
|
+
assert_equals 1 "$(get_linkcount file)"
|
|
377
|
+
cmp link.gz orig.gz
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
begin_test 'Multiple files'
|
|
381
|
+
cp file file2
|
|
382
|
+
gzip file file2
|
|
383
|
+
[ ! -e file ] && [ ! -e file2 ] && [ -e file.gz ] && [ -e file2.gz ]
|
|
384
|
+
gunzip file.gz file2.gz
|
|
385
|
+
[ -e file ] && [ -e file2 ] && [ ! -e file.gz ] && [ ! -e file2.gz ]
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
begin_test 'Multiple files, continue on warning'
|
|
389
|
+
mkdir 1
|
|
390
|
+
cp file 2
|
|
391
|
+
assert_skipped gzip 1 2
|
|
392
|
+
[ ! -e 1.gz ]
|
|
393
|
+
cmp file <(gunzip -c 2.gz)
|
|
394
|
+
rmdir 1
|
|
395
|
+
mkdir 1.gz
|
|
396
|
+
assert_skipped gunzip 1.gz 2.gz
|
|
397
|
+
[ ! -e 1 ]
|
|
398
|
+
cmp 2 file
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
if (( $(id -u) != 0 )); then
|
|
402
|
+
begin_test 'Multiple files, continue on error'
|
|
403
|
+
cp file 1
|
|
404
|
+
cp file 2
|
|
405
|
+
chmod a-r 1
|
|
406
|
+
assert_error 'Permission denied' gzip 1 2
|
|
407
|
+
[ ! -e 1.gz ]
|
|
408
|
+
cmp file <(gunzip -c 2.gz)
|
|
409
|
+
rm -f 1
|
|
410
|
+
cp 2.gz 1.gz
|
|
411
|
+
chmod a-r 1.gz
|
|
412
|
+
assert_error 'Permission denied' gunzip 1.gz 2.gz
|
|
413
|
+
[ ! -e 1 ]
|
|
414
|
+
cmp 2 file
|
|
415
|
+
fi
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
begin_test 'Compressing empty file'
|
|
419
|
+
echo -n > empty
|
|
420
|
+
gzip empty
|
|
421
|
+
gunzip empty.gz
|
|
422
|
+
cmp /dev/null empty
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
begin_test 'Decompressing malformed file'
|
|
426
|
+
echo -n > foo.gz
|
|
427
|
+
assert_error '\<(not in gzip format|unexpected end of file)\>' \
|
|
428
|
+
gunzip foo.gz
|
|
429
|
+
echo 1 > foo.gz
|
|
430
|
+
assert_error '\<not in gzip format\>' gunzip foo.gz
|
|
431
|
+
echo abcdefgh > foo.gz
|
|
432
|
+
assert_error '\<not in gzip format\>' gunzip foo.gz
|
|
433
|
+
echo -ne '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4b\x4c\x4a\x4e\x49\x24\x16\x73\x01\x00\x6c\x5b\xa2\x62\x2e\x00\x00\x00' \
|
|
434
|
+
> foo.gz
|
|
435
|
+
assert_error '\<(not in gzip format|crc error)\>' gunzip foo.gz
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
for suf in .foo foo .blaaaaaaaaaaaaaaaargh; do
|
|
439
|
+
begin_test "Custom suffix: $suf"
|
|
440
|
+
gzip -S $suf file
|
|
441
|
+
[ ! -e file ] && [ ! -e file.gz ] && [ -e file$suf ]
|
|
442
|
+
assert_skipped gunzip file$suf
|
|
443
|
+
gunzip -S $suf file$suf
|
|
444
|
+
[ -e file ] && [ ! -e file.gz ] && [ ! -e file$suf ]
|
|
445
|
+
done
|
|
446
|
+
# DIFFERENCE: GNU gzip lower cases suffix, we don't
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
begin_test 'Empty suffix is rejected'
|
|
450
|
+
assert_error '\<invalid suffix\>' gzip -S '""' file
|
|
451
|
+
assert_error '\<invalid suffix\>' gunzip -S '""' file
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
begin_test 'Timestamps and mode are preserved'
|
|
455
|
+
chmod 777 file
|
|
456
|
+
orig_stat=$(get_modeandtimestamps file)
|
|
457
|
+
gzip file
|
|
458
|
+
sleep 1
|
|
459
|
+
gunzip file.gz
|
|
460
|
+
assert_equals "$orig_stat" "$(get_modeandtimestamps file)"
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
begin_test 'Decompressing multi-member gzip file'
|
|
464
|
+
cat file file > orig
|
|
465
|
+
gzip -c file > file.gz
|
|
466
|
+
gzip -c file >> file.gz
|
|
467
|
+
gunzip -f file.gz
|
|
468
|
+
cmp file orig
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
begin_test 'Decompressing multi-member gzip file (final member smaller)'
|
|
472
|
+
echo 'hello world' > hello
|
|
473
|
+
cat file hello > orig
|
|
474
|
+
gzip -c file > file.gz
|
|
475
|
+
gzip -c hello >> file.gz
|
|
476
|
+
gunzip -f file.gz
|
|
477
|
+
cmp file orig
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
begin_test 'Help option'
|
|
481
|
+
gzip -h 2>&1 | grep -q 'Usage'
|
|
482
|
+
gunzip -h 2>&1 | grep -q 'Usage'
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
begin_test 'Incorrect usage'
|
|
486
|
+
for prog in gzip gunzip; do
|
|
487
|
+
for opt in '--invalid-option' '-0'; do
|
|
488
|
+
assert_error '\<(unrecognized|invalid) option\>' $prog $opt
|
|
489
|
+
done
|
|
490
|
+
done
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
begin_test '-t (test) option works'
|
|
494
|
+
good_files=(
|
|
495
|
+
'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsAAAA='
|
|
496
|
+
'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIoAUrnJFZVKqTkp+txAQBqzFDrLQAAAA==')
|
|
497
|
+
bad_files=(
|
|
498
|
+
'H4sIAO1YYmAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+46ehsAAAA='
|
|
499
|
+
'H4sIAO1YYmAAA3PMSVTITVTIzi85VABTIJ5jzpGZelwAX+86ehsAAAA='
|
|
500
|
+
'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsBAAA='
|
|
501
|
+
'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIogUrnJFZVKqTkp+txAQBqzFDrLQAAAA=='
|
|
502
|
+
'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9L')
|
|
503
|
+
for contents in "${good_files[@]}"; do
|
|
504
|
+
echo "$contents" | base64 -d | gzip -t
|
|
505
|
+
done
|
|
506
|
+
for contents in "${bad_files[@]}"; do
|
|
507
|
+
echo "$contents" | base64 -d > file
|
|
508
|
+
assert_error '\<invalid compressed data|file corrupt|unexpected end of file|Out of memory\>' \
|
|
509
|
+
gzip -t file
|
|
510
|
+
done
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
begin_test '-q (quiet) option works'
|
|
514
|
+
mkdir dir
|
|
515
|
+
gunzip -q dir &> output || true
|
|
516
|
+
[ ! -s output ]
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
begin_test 'Version information'
|
|
520
|
+
gzip -V | grep -q Copyright
|
|
521
|
+
gunzip -V | grep -q Copyright
|
|
522
|
+
|
|
523
|
+
CURRENT_TEST=
|