zopfli 0.0.2 → 0.0.8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +6 -14
  2. data/.gitmodules +1 -1
  3. data/.travis.yml +23 -0
  4. data/Gemfile +2 -0
  5. data/README.md +6 -1
  6. data/Rakefile +8 -10
  7. data/ext/extconf.rb +2 -1
  8. data/ext/zopfli.c +39 -20
  9. data/lib/zopfli/version.rb +1 -1
  10. data/smoke.sh +9 -0
  11. data/{test → spec}/fixtures/alice29.txt +0 -0
  12. data/spec/spec_helper.rb +2 -0
  13. data/spec/zopfli_spec.rb +68 -0
  14. data/vendor/zopfli/{blocksplitter.c → src/zopfli/blocksplitter.c} +41 -53
  15. data/vendor/zopfli/{blocksplitter.h → src/zopfli/blocksplitter.h} +2 -6
  16. data/vendor/zopfli/{cache.c → src/zopfli/cache.c} +6 -0
  17. data/vendor/zopfli/{cache.h → src/zopfli/cache.h} +0 -0
  18. data/vendor/zopfli/src/zopfli/deflate.c +931 -0
  19. data/vendor/zopfli/{deflate.h → src/zopfli/deflate.h} +17 -2
  20. data/vendor/zopfli/src/zopfli/gzip_container.c +124 -0
  21. data/vendor/zopfli/{gzip_container.h → src/zopfli/gzip_container.h} +8 -0
  22. data/vendor/zopfli/{hash.c → src/zopfli/hash.c} +18 -10
  23. data/vendor/zopfli/{hash.h → src/zopfli/hash.h} +10 -7
  24. data/vendor/zopfli/{katajainen.c → src/zopfli/katajainen.c} +73 -62
  25. data/vendor/zopfli/{katajainen.h → src/zopfli/katajainen.h} +1 -1
  26. data/vendor/zopfli/{lz77.c → src/zopfli/lz77.c} +190 -42
  27. data/vendor/zopfli/{lz77.h → src/zopfli/lz77.h} +39 -23
  28. data/vendor/zopfli/{squeeze.c → src/zopfli/squeeze.c} +75 -61
  29. data/vendor/zopfli/{squeeze.h → src/zopfli/squeeze.h} +1 -0
  30. data/vendor/zopfli/{util.c → src/zopfli/symbols.h} +49 -23
  31. data/vendor/zopfli/{tree.c → src/zopfli/tree.c} +0 -0
  32. data/vendor/zopfli/{tree.h → src/zopfli/tree.h} +0 -0
  33. data/vendor/zopfli/src/zopfli/util.c +35 -0
  34. data/vendor/zopfli/{util.h → src/zopfli/util.h} +6 -23
  35. data/vendor/zopfli/{zlib_container.c → src/zopfli/zlib_container.c} +1 -1
  36. data/vendor/zopfli/{zlib_container.h → src/zopfli/zlib_container.h} +8 -0
  37. data/vendor/zopfli/{zopfli.h → src/zopfli/zopfli.h} +10 -4
  38. data/vendor/zopfli/{zopfli_bin.c → src/zopfli/zopfli_bin.c} +31 -15
  39. data/vendor/zopfli/{zopfli_lib.c → src/zopfli/zopfli_lib.c} +1 -2
  40. data/zopfli.gemspec +9 -28
  41. metadata +51 -50
  42. data/test/test_zopfli_deflate.rb +0 -47
  43. data/vendor/zopfli/CONTRIBUTORS +0 -6
  44. data/vendor/zopfli/README +0 -25
  45. data/vendor/zopfli/deflate.c +0 -698
  46. data/vendor/zopfli/gzip_container.c +0 -117
  47. data/vendor/zopfli/makefile +0 -5
metadata CHANGED
@@ -1,55 +1,55 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zopfli
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - miyucy
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-05-09 00:00:00.000000000 Z
11
+ date: 2020-10-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.3'
19
+ version: 2.1.4
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.3'
26
+ version: 2.1.4
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ! '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ! '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: minitest
42
+ name: rspec
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ! '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ! '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  description: zopfli
@@ -60,8 +60,9 @@ extensions:
60
60
  - ext/extconf.rb
61
61
  extra_rdoc_files: []
62
62
  files:
63
- - .gitignore
64
- - .gitmodules
63
+ - ".gitignore"
64
+ - ".gitmodules"
65
+ - ".travis.yml"
65
66
  - Gemfile
66
67
  - LICENSE.txt
67
68
  - README.md
@@ -69,38 +70,38 @@ files:
69
70
  - ext/extconf.rb
70
71
  - ext/zopfli.c
71
72
  - lib/zopfli/version.rb
72
- - test/fixtures/alice29.txt
73
- - test/test_zopfli_deflate.rb
74
- - zopfli.gemspec
75
- - vendor/zopfli/CONTRIBUTORS
73
+ - smoke.sh
74
+ - spec/fixtures/alice29.txt
75
+ - spec/spec_helper.rb
76
+ - spec/zopfli_spec.rb
76
77
  - vendor/zopfli/COPYING
77
- - vendor/zopfli/README
78
- - vendor/zopfli/blocksplitter.c
79
- - vendor/zopfli/blocksplitter.h
80
- - vendor/zopfli/cache.c
81
- - vendor/zopfli/cache.h
82
- - vendor/zopfli/deflate.c
83
- - vendor/zopfli/deflate.h
84
- - vendor/zopfli/gzip_container.c
85
- - vendor/zopfli/gzip_container.h
86
- - vendor/zopfli/hash.c
87
- - vendor/zopfli/hash.h
88
- - vendor/zopfli/katajainen.c
89
- - vendor/zopfli/katajainen.h
90
- - vendor/zopfli/lz77.c
91
- - vendor/zopfli/lz77.h
92
- - vendor/zopfli/makefile
93
- - vendor/zopfli/squeeze.c
94
- - vendor/zopfli/squeeze.h
95
- - vendor/zopfli/tree.c
96
- - vendor/zopfli/tree.h
97
- - vendor/zopfli/util.c
98
- - vendor/zopfli/util.h
99
- - vendor/zopfli/zlib_container.c
100
- - vendor/zopfli/zlib_container.h
101
- - vendor/zopfli/zopfli.h
102
- - vendor/zopfli/zopfli_bin.c
103
- - vendor/zopfli/zopfli_lib.c
78
+ - vendor/zopfli/src/zopfli/blocksplitter.c
79
+ - vendor/zopfli/src/zopfli/blocksplitter.h
80
+ - vendor/zopfli/src/zopfli/cache.c
81
+ - vendor/zopfli/src/zopfli/cache.h
82
+ - vendor/zopfli/src/zopfli/deflate.c
83
+ - vendor/zopfli/src/zopfli/deflate.h
84
+ - vendor/zopfli/src/zopfli/gzip_container.c
85
+ - vendor/zopfli/src/zopfli/gzip_container.h
86
+ - vendor/zopfli/src/zopfli/hash.c
87
+ - vendor/zopfli/src/zopfli/hash.h
88
+ - vendor/zopfli/src/zopfli/katajainen.c
89
+ - vendor/zopfli/src/zopfli/katajainen.h
90
+ - vendor/zopfli/src/zopfli/lz77.c
91
+ - vendor/zopfli/src/zopfli/lz77.h
92
+ - vendor/zopfli/src/zopfli/squeeze.c
93
+ - vendor/zopfli/src/zopfli/squeeze.h
94
+ - vendor/zopfli/src/zopfli/symbols.h
95
+ - vendor/zopfli/src/zopfli/tree.c
96
+ - vendor/zopfli/src/zopfli/tree.h
97
+ - vendor/zopfli/src/zopfli/util.c
98
+ - vendor/zopfli/src/zopfli/util.h
99
+ - vendor/zopfli/src/zopfli/zlib_container.c
100
+ - vendor/zopfli/src/zopfli/zlib_container.h
101
+ - vendor/zopfli/src/zopfli/zopfli.h
102
+ - vendor/zopfli/src/zopfli/zopfli_bin.c
103
+ - vendor/zopfli/src/zopfli/zopfli_lib.c
104
+ - zopfli.gemspec
104
105
  homepage: http://github.com/miyucy/zopfli
105
106
  licenses:
106
107
  - MIT
@@ -111,20 +112,20 @@ require_paths:
111
112
  - lib
112
113
  required_ruby_version: !ruby/object:Gem::Requirement
113
114
  requirements:
114
- - - ! '>='
115
+ - - ">="
115
116
  - !ruby/object:Gem::Version
116
117
  version: '0'
117
118
  required_rubygems_version: !ruby/object:Gem::Requirement
118
119
  requirements:
119
- - - ! '>='
120
+ - - ">="
120
121
  - !ruby/object:Gem::Version
121
122
  version: '0'
122
123
  requirements: []
123
- rubyforge_project:
124
- rubygems_version: 2.0.3
124
+ rubygems_version: 3.0.8
125
125
  signing_key:
126
126
  specification_version: 4
127
127
  summary: zopfli
128
128
  test_files:
129
- - test/fixtures/alice29.txt
130
- - test/test_zopfli_deflate.rb
129
+ - spec/fixtures/alice29.txt
130
+ - spec/spec_helper.rb
131
+ - spec/zopfli_spec.rb
@@ -1,47 +0,0 @@
1
- require "minitest/spec"
2
- require "minitest/autorun"
3
- require "zopfli"
4
- require "stringio"
5
- require "zlib"
6
-
7
- describe Zopfli do
8
- it "works fine" do
9
- fixture = fixtures("alice29.txt").read
10
-
11
- Zopfli.deflate(fixture).must_equal(Zopfli.deflate(fixture, format: :zlib))
12
- end
13
-
14
- it "zlib format works" do
15
- fixture = fixtures("alice29.txt").read
16
-
17
- deflated = Zopfli.deflate fixture, format: :zlib
18
-
19
- uncompressed = Zlib::Inflate.inflate deflated
20
-
21
- uncompressed.must_equal fixture
22
- end
23
-
24
- it "gzip format works" do
25
- fixture = fixtures("alice29.txt").read
26
-
27
- gzipped = Zopfli.deflate fixture, format: :gzip
28
-
29
- uncompressed = nil
30
- Zlib::GzipReader.wrap(StringIO.new gzipped) { |gz|
31
- uncompressed = gz.read
32
- }
33
-
34
- uncompressed.must_equal fixture
35
- end
36
-
37
- it "deflate format works" do
38
- fixture = fixtures("alice29.txt").read
39
-
40
- deflate = Zopfli.deflate fixture, format: :deflate
41
- skip "How to test"
42
- end
43
-
44
- def fixtures(name)
45
- File.open(File.join File.dirname(__FILE__), "fixtures", name)
46
- end
47
- end
@@ -1,6 +0,0 @@
1
- Mark Adler
2
- Jyrki Alakuijala
3
- Daniel Reed
4
- Huzaifa Sidhpurwala
5
- Péter Szabó
6
- Lode Vandevenne
@@ -1,25 +0,0 @@
1
- Zopfli Compression Algorithm is a compression library programmed in C to perform
2
- very good, but slow, deflate or zlib compression.
3
-
4
- zopfli.c is separate from the library and contains an example program to create
5
- very well compressed gzip files.
6
-
7
- The basic functions to compress data are ZopfliDeflate in deflate.h,
8
- ZopfliZlibCompress in zlib_container.h and ZopfliGzipCompress in
9
- gzip_container.h, or ZopfliCompress in zopfli.h.
10
- Use the ZopfliOptions object to set parameters that affect the speed and
11
- compression. Use the ZopfliInitOptions function to place the default values in
12
- the ZopfliOptions first.
13
-
14
- Deflate creates a valid deflate stream in memory, see:
15
- http://www.ietf.org/rfc/rfc1951.txt
16
- ZlibCompress creates a valid zlib stream in memory, see:
17
- http://www.ietf.org/rfc/rfc1950.txt
18
- GzipCompress creates a valid gzip stream in memory, see:
19
- http://www.ietf.org/rfc/rfc1952.txt
20
-
21
- This library can only compress, not decompress. Existing zlib or deflate
22
- libraries can decompress the data.
23
-
24
- Zopfli Compression Algorithm was created by Lode Vandevenne and Jyrki
25
- Alakuijala, based on an algorithm by Jyrki Alakuijala.
@@ -1,698 +0,0 @@
1
- /*
2
- Copyright 2011 Google Inc. All Rights Reserved.
3
-
4
- Licensed under the Apache License, Version 2.0 (the "License");
5
- you may not use this file except in compliance with the License.
6
- You may obtain a copy of the License at
7
-
8
- http://www.apache.org/licenses/LICENSE-2.0
9
-
10
- Unless required by applicable law or agreed to in writing, software
11
- distributed under the License is distributed on an "AS IS" BASIS,
12
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- See the License for the specific language governing permissions and
14
- limitations under the License.
15
-
16
- Author: lode.vandevenne@gmail.com (Lode Vandevenne)
17
- Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18
- */
19
-
20
- #include "deflate.h"
21
-
22
- #include <assert.h>
23
- #include <stdio.h>
24
- #include <stdlib.h>
25
-
26
- #include "blocksplitter.h"
27
- #include "lz77.h"
28
- #include "squeeze.h"
29
- #include "tree.h"
30
-
31
- static void AddBit(int bit,
32
- unsigned char* bp, unsigned char** out, size_t* outsize) {
33
- if (((*bp) & 7) == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
34
- (*out)[*outsize - 1] |= bit << ((*bp) & 7);
35
- (*bp)++;
36
- }
37
-
38
- static void AddBits(unsigned symbol, unsigned length,
39
- unsigned char* bp, unsigned char** out, size_t* outsize) {
40
- /* TODO(lode): make more efficient (add more bits at once). */
41
- unsigned i;
42
- for (i = 0; i < length; i++) {
43
- unsigned bit = (symbol >> i) & 1;
44
- if (((*bp) & 7) == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
45
- (*out)[*outsize - 1] |= bit << ((*bp) & 7);
46
- (*bp)++;
47
- }
48
- }
49
-
50
- /*
51
- Adds bits, like AddBits, but the order is inverted. The deflate specification
52
- uses both orders in one standard.
53
- */
54
- static void AddHuffmanBits(unsigned symbol, unsigned length,
55
- unsigned char* bp, unsigned char** out,
56
- size_t* outsize) {
57
- /* TODO(lode): make more efficient (add more bits at once). */
58
- unsigned i;
59
- for (i = 0; i < length; i++) {
60
- unsigned bit = (symbol >> (length - i - 1)) & 1;
61
- if (((*bp) & 7) == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
62
- (*out)[*outsize - 1] |= bit << ((*bp) & 7);
63
- (*bp)++;
64
- }
65
- }
66
-
67
- /*
68
- Ensures there are at least 2 distance codes to support buggy decoders.
69
- Zlib 1.2.1 and below have a bug where it fails if there isn't at least 1
70
- distance code (with length > 0), even though it's valid according to the
71
- deflate spec to have 0 distance codes. On top of that, some mobile phones
72
- require at least two distance codes. To support these decoders too (but
73
- potentially at the cost of a few bytes), add dummy code lengths of 1.
74
- References to this bug can be found in the changelog of
75
- Zlib 1.2.2 and here: http://www.jonof.id.au/forum/index.php?topic=515.0.
76
-
77
- d_lengths: the 32 lengths of the distance codes.
78
- */
79
- static void PatchDistanceCodesForBuggyDecoders(unsigned* d_lengths) {
80
- int num_dist_codes = 0; /* Amount of non-zero distance codes */
81
- int i;
82
- for (i = 0; i < 30 /* Ignore the two unused codes from the spec */; i++) {
83
- if (d_lengths[i]) num_dist_codes++;
84
- if (num_dist_codes >= 2) return; /* Two or more codes is fine. */
85
- }
86
-
87
- if (num_dist_codes == 0) {
88
- d_lengths[0] = d_lengths[1] = 1;
89
- } else if (num_dist_codes == 1) {
90
- d_lengths[d_lengths[0] ? 1 : 0] = 1;
91
- }
92
- }
93
-
94
- static void AddDynamicTree(const unsigned* ll_lengths,
95
- const unsigned* d_lengths,
96
- unsigned char* bp,
97
- unsigned char** out, size_t* outsize) {
98
- unsigned* lld_lengths = 0; /* All litlen and dist lengthts with ending zeros
99
- trimmed together in one array. */
100
- unsigned lld_total; /* Size of lld_lengths. */
101
- unsigned* rle = 0; /* Runlength encoded version of lengths of litlen and dist
102
- trees. */
103
- unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */
104
- size_t rle_size = 0; /* Size of rle array. */
105
- size_t rle_bits_size = 0; /* Should have same value as rle_size. */
106
- unsigned hlit = 29; /* 286 - 257 */
107
- unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/
108
- unsigned hclen;
109
- size_t i, j;
110
- size_t clcounts[19];
111
- unsigned clcl[19]; /* Code length code lengths. */
112
- unsigned clsymbols[19];
113
- /* The order in which code length code lengths are encoded as per deflate. */
114
- unsigned order[19] = {
115
- 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
116
- };
117
-
118
- /* Trim zeros. */
119
- while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--;
120
- while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--;
121
-
122
- lld_total = hlit + 257 + hdist + 1;
123
- lld_lengths = (unsigned*)malloc(sizeof(*lld_lengths) * lld_total);
124
- if (!lld_lengths) exit(-1); /* Allocation failed. */
125
-
126
- for (i = 0; i < lld_total; i++) {
127
- lld_lengths[i] = i < 257 + hlit
128
- ? ll_lengths[i] : d_lengths[i - 257 - hlit];
129
- assert(lld_lengths[i] < 16);
130
- }
131
-
132
- for (i = 0; i < lld_total; i++) {
133
- size_t count = 0;
134
- for (j = i; j < lld_total && lld_lengths[i] == lld_lengths[j]; j++) {
135
- count++;
136
- }
137
- if (count >= 4 || (count >= 3 && lld_lengths[i] == 0)) {
138
- if (lld_lengths[i] == 0) {
139
- if (count > 10) {
140
- if (count > 138) count = 138;
141
- ZOPFLI_APPEND_DATA(18, &rle, &rle_size);
142
- ZOPFLI_APPEND_DATA(count - 11, &rle_bits, &rle_bits_size);
143
- } else {
144
- ZOPFLI_APPEND_DATA(17, &rle, &rle_size);
145
- ZOPFLI_APPEND_DATA(count - 3, &rle_bits, &rle_bits_size);
146
- }
147
- } else {
148
- unsigned repeat = count - 1; /* Since the first one is hardcoded. */
149
- ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
150
- ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
151
- while (repeat >= 6) {
152
- ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
153
- ZOPFLI_APPEND_DATA(6 - 3, &rle_bits, &rle_bits_size);
154
- repeat -= 6;
155
- }
156
- if (repeat >= 3) {
157
- ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
158
- ZOPFLI_APPEND_DATA(3 - 3, &rle_bits, &rle_bits_size);
159
- repeat -= 3;
160
- }
161
- while (repeat != 0) {
162
- ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
163
- ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
164
- repeat--;
165
- }
166
- }
167
-
168
- i += count - 1;
169
- } else {
170
- ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
171
- ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
172
- }
173
- assert(rle[rle_size - 1] <= 18);
174
- }
175
-
176
- for (i = 0; i < 19; i++) {
177
- clcounts[i] = 0;
178
- }
179
- for (i = 0; i < rle_size; i++) {
180
- clcounts[rle[i]]++;
181
- }
182
-
183
- ZopfliCalculateBitLengths(clcounts, 19, 7, clcl);
184
- ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols);
185
-
186
- hclen = 15;
187
- /* Trim zeros. */
188
- while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--;
189
-
190
- AddBits(hlit, 5, bp, out, outsize);
191
- AddBits(hdist, 5, bp, out, outsize);
192
- AddBits(hclen, 4, bp, out, outsize);
193
-
194
- for (i = 0; i < hclen + 4; i++) {
195
- AddBits(clcl[order[i]], 3, bp, out, outsize);
196
- }
197
-
198
- for (i = 0; i < rle_size; i++) {
199
- unsigned symbol = clsymbols[rle[i]];
200
- AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize);
201
- /* Extra bits. */
202
- if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize);
203
- else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize);
204
- else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize);
205
- }
206
-
207
- free(lld_lengths);
208
- free(rle);
209
- free(rle_bits);
210
- }
211
-
212
- /*
213
- Gives the exact size of the tree, in bits, as it will be encoded in DEFLATE.
214
- */
215
- static size_t CalculateTreeSize(const unsigned* ll_lengths,
216
- const unsigned* d_lengths,
217
- size_t* ll_counts, size_t* d_counts) {
218
- unsigned char* dummy = 0;
219
- size_t dummysize = 0;
220
- unsigned char bp = 0;
221
-
222
- (void)ll_counts;
223
- (void)d_counts;
224
-
225
- AddDynamicTree(ll_lengths, d_lengths, &bp, &dummy, &dummysize);
226
- free(dummy);
227
-
228
- return dummysize * 8 + (bp & 7);
229
- }
230
-
231
- /*
232
- Adds all lit/len and dist codes from the lists as huffman symbols. Does not add
233
- end code 256. expected_data_size is the uncompressed block size, used for
234
- assert, but you can set it to 0 to not do the assertion.
235
- */
236
- static void AddLZ77Data(const unsigned short* litlens,
237
- const unsigned short* dists,
238
- size_t lstart, size_t lend,
239
- size_t expected_data_size,
240
- const unsigned* ll_symbols, const unsigned* ll_lengths,
241
- const unsigned* d_symbols, const unsigned* d_lengths,
242
- unsigned char* bp,
243
- unsigned char** out, size_t* outsize) {
244
- size_t testlength = 0;
245
- size_t i;
246
-
247
- for (i = lstart; i < lend; i++) {
248
- unsigned dist = dists[i];
249
- unsigned litlen = litlens[i];
250
- if (dist == 0) {
251
- assert(litlen < 256);
252
- assert(ll_lengths[litlen] > 0);
253
- AddHuffmanBits(ll_symbols[litlen], ll_lengths[litlen], bp, out, outsize);
254
- testlength++;
255
- } else {
256
- unsigned lls = ZopfliGetLengthSymbol(litlen);
257
- unsigned ds = ZopfliGetDistSymbol(dist);
258
- assert(litlen >= 3 && litlen <= 288);
259
- assert(ll_lengths[lls] > 0);
260
- assert(d_lengths[ds] > 0);
261
- AddHuffmanBits(ll_symbols[lls], ll_lengths[lls], bp, out, outsize);
262
- AddBits(ZopfliGetLengthExtraBitsValue(litlen),
263
- ZopfliGetLengthExtraBits(litlen),
264
- bp, out, outsize);
265
- AddHuffmanBits(d_symbols[ds], d_lengths[ds], bp, out, outsize);
266
- AddBits(ZopfliGetDistExtraBitsValue(dist),
267
- ZopfliGetDistExtraBits(dist),
268
- bp, out, outsize);
269
- testlength += litlen;
270
- }
271
- }
272
- assert(expected_data_size == 0 || testlength == expected_data_size);
273
- }
274
-
275
- static void GetFixedTree(unsigned* ll_lengths, unsigned* d_lengths) {
276
- size_t i;
277
- for (i = 0; i < 144; i++) ll_lengths[i] = 8;
278
- for (i = 144; i < 256; i++) ll_lengths[i] = 9;
279
- for (i = 256; i < 280; i++) ll_lengths[i] = 7;
280
- for (i = 280; i < 288; i++) ll_lengths[i] = 8;
281
- for (i = 0; i < 32; i++) d_lengths[i] = 5;
282
- }
283
-
284
- /*
285
- Calculates size of the part after the header and tree of an LZ77 block, in bits.
286
- */
287
- static size_t CalculateBlockSymbolSize(const unsigned* ll_lengths,
288
- const unsigned* d_lengths,
289
- const unsigned short* litlens,
290
- const unsigned short* dists,
291
- size_t lstart, size_t lend) {
292
- size_t result = 0;
293
- size_t i;
294
- for (i = lstart; i < lend; i++) {
295
- if (dists[i] == 0) {
296
- result += ll_lengths[litlens[i]];
297
- } else {
298
- result += ll_lengths[ZopfliGetLengthSymbol(litlens[i])];
299
- result += d_lengths[ZopfliGetDistSymbol(dists[i])];
300
- result += ZopfliGetLengthExtraBits(litlens[i]);
301
- result += ZopfliGetDistExtraBits(dists[i]);
302
- }
303
- }
304
- result += ll_lengths[256]; /*end symbol*/
305
- return result;
306
- }
307
-
308
- double ZopfliCalculateBlockSize(const unsigned short* litlens,
309
- const unsigned short* dists,
310
- size_t lstart, size_t lend, int btype) {
311
- size_t ll_counts[288];
312
- size_t d_counts[32];
313
-
314
- unsigned ll_lengths[288];
315
- unsigned d_lengths[32];
316
-
317
- double result = 3; /*bfinal and btype bits*/
318
-
319
- assert(btype == 1 || btype == 2); /* This is not for uncompressed blocks. */
320
-
321
- if(btype == 1) {
322
- GetFixedTree(ll_lengths, d_lengths);
323
- } else {
324
- ZopfliLZ77Counts(litlens, dists, lstart, lend, ll_counts, d_counts);
325
- ZopfliCalculateBitLengths(ll_counts, 288, 15, ll_lengths);
326
- ZopfliCalculateBitLengths(d_counts, 32, 15, d_lengths);
327
- PatchDistanceCodesForBuggyDecoders(d_lengths);
328
- result += CalculateTreeSize(ll_lengths, d_lengths, ll_counts, d_counts);
329
- }
330
-
331
- result += CalculateBlockSymbolSize(
332
- ll_lengths, d_lengths, litlens, dists, lstart, lend);
333
-
334
- return result;
335
- }
336
-
337
- /*
338
- Adds a deflate block with the given LZ77 data to the output.
339
- options: global program options
340
- btype: the block type, must be 1 or 2
341
- final: whether to set the "final" bit on this block, must be the last block
342
- litlens: literal/length array of the LZ77 data, in the same format as in
343
- ZopfliLZ77Store.
344
- dists: distance array of the LZ77 data, in the same format as in
345
- ZopfliLZ77Store.
346
- lstart: where to start in the LZ77 data
347
- lend: where to end in the LZ77 data (not inclusive)
348
- expected_data_size: the uncompressed block size, used for assert, but you can
349
- set it to 0 to not do the assertion.
350
- bp: output bit pointer
351
- out: dynamic output array to append to
352
- outsize: dynamic output array size
353
- */
354
- static void AddLZ77Block(const ZopfliOptions* options, int btype, int final,
355
- const unsigned short* litlens,
356
- const unsigned short* dists,
357
- size_t lstart, size_t lend,
358
- size_t expected_data_size,
359
- unsigned char* bp, unsigned char** out, size_t* outsize) {
360
- size_t ll_counts[288];
361
- size_t d_counts[32];
362
- unsigned ll_lengths[288];
363
- unsigned d_lengths[32];
364
- unsigned ll_symbols[288];
365
- unsigned d_symbols[32];
366
- size_t detect_block_size = *outsize;
367
- size_t compressed_size;
368
- size_t uncompressed_size = 0;
369
- size_t i;
370
-
371
- AddBit(final, bp, out, outsize);
372
- AddBit(btype & 1, bp, out, outsize);
373
- AddBit((btype & 2) >> 1, bp, out, outsize);
374
-
375
- if (btype == 1) {
376
- /* Fixed block. */
377
- GetFixedTree(ll_lengths, d_lengths);
378
- } else {
379
- /* Dynamic block. */
380
- unsigned detect_tree_size;
381
- assert(btype == 2);
382
- ZopfliLZ77Counts(litlens, dists, lstart, lend, ll_counts, d_counts);
383
- ZopfliCalculateBitLengths(ll_counts, 288, 15, ll_lengths);
384
- ZopfliCalculateBitLengths(d_counts, 32, 15, d_lengths);
385
- PatchDistanceCodesForBuggyDecoders(d_lengths);
386
- detect_tree_size = *outsize;
387
- AddDynamicTree(ll_lengths, d_lengths, bp, out, outsize);
388
- if (options->verbose) {
389
- fprintf(stderr, "treesize: %d\n", (int)(*outsize - detect_tree_size));
390
- }
391
-
392
- /* Assert that for every present symbol, the code length is non-zero. */
393
- /* TODO(lode): remove this in release version. */
394
- for (i = 0; i < 288; i++) assert(ll_counts[i] == 0 || ll_lengths[i] > 0);
395
- for (i = 0; i < 32; i++) assert(d_counts[i] == 0 || d_lengths[i] > 0);
396
- }
397
-
398
- ZopfliLengthsToSymbols(ll_lengths, 288, 15, ll_symbols);
399
- ZopfliLengthsToSymbols(d_lengths, 32, 15, d_symbols);
400
-
401
- detect_block_size = *outsize;
402
- AddLZ77Data(litlens, dists, lstart, lend, expected_data_size,
403
- ll_symbols, ll_lengths, d_symbols, d_lengths,
404
- bp, out, outsize);
405
- /* End symbol. */
406
- AddHuffmanBits(ll_symbols[256], ll_lengths[256], bp, out, outsize);
407
-
408
- for (i = lstart; i < lend; i++) {
409
- uncompressed_size += dists[i] == 0 ? 1 : litlens[i];
410
- }
411
- compressed_size = *outsize - detect_block_size;
412
- if (options->verbose) {
413
- fprintf(stderr, "compressed block size: %d (%dk) (unc: %d)\n",
414
- (int)compressed_size, (int)(compressed_size / 1024),
415
- (int)(uncompressed_size));
416
- }
417
- }
418
-
419
- static void DeflateDynamicBlock(const ZopfliOptions* options, int final,
420
- const unsigned char* in,
421
- size_t instart, size_t inend,
422
- unsigned char* bp,
423
- unsigned char** out, size_t* outsize) {
424
- ZopfliBlockState s;
425
- size_t blocksize = inend - instart;
426
- ZopfliLZ77Store store;
427
- int btype = 2;
428
-
429
- ZopfliInitLZ77Store(&store);
430
-
431
- s.options = options;
432
- s.blockstart = instart;
433
- s.blockend = inend;
434
- #ifdef ZOPFLI_LONGEST_MATCH_CACHE
435
- s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache));
436
- ZopfliInitCache(blocksize, s.lmc);
437
- #endif
438
-
439
- ZopfliLZ77Optimal(&s, in, instart, inend, &store);
440
-
441
- /* For small block, encoding with fixed tree can be smaller. For large block,
442
- don't bother doing this expensive test, dynamic tree will be better.*/
443
- if (store.size < 1000) {
444
- double dyncost, fixedcost;
445
- ZopfliLZ77Store fixedstore;
446
- ZopfliInitLZ77Store(&fixedstore);
447
- ZopfliLZ77OptimalFixed(&s, in, instart, inend, &fixedstore);
448
- dyncost = ZopfliCalculateBlockSize(store.litlens, store.dists,
449
- 0, store.size, 2);
450
- fixedcost = ZopfliCalculateBlockSize(fixedstore.litlens, fixedstore.dists,
451
- 0, fixedstore.size, 1);
452
- if (fixedcost < dyncost) {
453
- btype = 1;
454
- ZopfliCleanLZ77Store(&store);
455
- store = fixedstore;
456
- } else {
457
- ZopfliCleanLZ77Store(&fixedstore);
458
- }
459
- }
460
-
461
- AddLZ77Block(s.options, btype, final,
462
- store.litlens, store.dists, 0, store.size,
463
- blocksize, bp, out, outsize);
464
-
465
- #ifdef ZOPFLI_LONGEST_MATCH_CACHE
466
- ZopfliCleanCache(s.lmc);
467
- free(s.lmc);
468
- #endif
469
- ZopfliCleanLZ77Store(&store);
470
- }
471
-
472
- static void DeflateFixedBlock(const ZopfliOptions* options, int final,
473
- const unsigned char* in,
474
- size_t instart, size_t inend,
475
- unsigned char* bp,
476
- unsigned char** out, size_t* outsize) {
477
- ZopfliBlockState s;
478
- size_t blocksize = inend - instart;
479
- ZopfliLZ77Store store;
480
-
481
- ZopfliInitLZ77Store(&store);
482
-
483
- s.options = options;
484
- s.blockstart = instart;
485
- s.blockend = inend;
486
- #ifdef ZOPFLI_LONGEST_MATCH_CACHE
487
- s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache));
488
- ZopfliInitCache(blocksize, s.lmc);
489
- #endif
490
-
491
- ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store);
492
-
493
- AddLZ77Block(s.options, 1, final, store.litlens, store.dists, 0, store.size,
494
- blocksize, bp, out, outsize);
495
-
496
- #ifdef ZOPFLI_LONGEST_MATCH_CACHE
497
- ZopfliCleanCache(s.lmc);
498
- free(s.lmc);
499
- #endif
500
- ZopfliCleanLZ77Store(&store);
501
- }
502
-
503
- static void DeflateNonCompressedBlock(const ZopfliOptions* options, int final,
504
- const unsigned char* in, size_t instart,
505
- size_t inend,
506
- unsigned char* bp,
507
- unsigned char** out, size_t* outsize) {
508
- size_t i;
509
- size_t blocksize = inend - instart;
510
- unsigned short nlen = ~blocksize;
511
-
512
- (void)options;
513
- assert(blocksize < 65536); /* Non compressed blocks are max this size. */
514
-
515
- AddBit(final, bp, out, outsize);
516
- /* BTYPE 00 */
517
- AddBit(0, bp, out, outsize);
518
- AddBit(0, bp, out, outsize);
519
-
520
- /* Any bits of input up to the next byte boundary are ignored. */
521
- *bp = 0;
522
-
523
- ZOPFLI_APPEND_DATA(blocksize % 256, out, outsize);
524
- ZOPFLI_APPEND_DATA((blocksize / 256) % 256, out, outsize);
525
- ZOPFLI_APPEND_DATA(nlen % 256, out, outsize);
526
- ZOPFLI_APPEND_DATA((nlen / 256) % 256, out, outsize);
527
-
528
- for (i = instart; i < inend; i++) {
529
- ZOPFLI_APPEND_DATA(in[i], out, outsize);
530
- }
531
- }
532
-
533
- static void DeflateBlock(const ZopfliOptions* options,
534
- int btype, int final,
535
- const unsigned char* in, size_t instart, size_t inend,
536
- unsigned char* bp,
537
- unsigned char** out, size_t* outsize) {
538
- if (btype == 0) {
539
- DeflateNonCompressedBlock(
540
- options, final, in, instart, inend, bp, out, outsize);
541
- } else if (btype == 1) {
542
- DeflateFixedBlock(options, final, in, instart, inend, bp, out, outsize);
543
- } else {
544
- assert (btype == 2);
545
- DeflateDynamicBlock(options, final, in, instart, inend, bp, out, outsize);
546
- }
547
- }
548
-
549
- /*
550
- Does squeeze strategy where first block splitting is done, then each block is
551
- squeezed.
552
- Parameters: see description of the ZopfliDeflate function.
553
- */
554
- static void DeflateSplittingFirst(const ZopfliOptions* options,
555
- int btype, int final,
556
- const unsigned char* in,
557
- size_t instart, size_t inend,
558
- unsigned char* bp,
559
- unsigned char** out, size_t* outsize) {
560
- size_t i;
561
- size_t* splitpoints = 0;
562
- size_t npoints = 0;
563
- if (btype == 0) {
564
- ZopfliBlockSplitSimple(in, instart, inend, 65535, &splitpoints, &npoints);
565
- } else if (btype == 1) {
566
- /* If all blocks are fixed tree, splitting into separate blocks only
567
- increases the total size. Leave npoints at 0, this represents 1 block. */
568
- } else {
569
- ZopfliBlockSplit(options, in, instart, inend,
570
- options->blocksplittingmax, &splitpoints, &npoints);
571
- }
572
-
573
- for (i = 0; i <= npoints; i++) {
574
- size_t start = i == 0 ? instart : splitpoints[i - 1];
575
- size_t end = i == npoints ? inend : splitpoints[i];
576
- DeflateBlock(options, btype, i == npoints && final, in, start, end,
577
- bp, out, outsize);
578
- }
579
-
580
- free(splitpoints);
581
- }
582
-
583
- /*
584
- Does squeeze strategy where first the best possible lz77 is done, and then based
585
- on that data, block splitting is done.
586
- Parameters: see description of the ZopfliDeflate function.
587
- */
588
- static void DeflateSplittingLast(const ZopfliOptions* options,
589
- int btype, int final,
590
- const unsigned char* in,
591
- size_t instart, size_t inend,
592
- unsigned char* bp,
593
- unsigned char** out, size_t* outsize) {
594
- size_t i;
595
- ZopfliBlockState s;
596
- ZopfliLZ77Store store;
597
- size_t* splitpoints = 0;
598
- size_t npoints = 0;
599
-
600
- if (btype == 0) {
601
- /* This function only supports LZ77 compression. DeflateSplittingFirst
602
- supports the special case of noncompressed data. Punt it to that one. */
603
- DeflateSplittingFirst(options, btype, final,
604
- in, instart, inend,
605
- bp, out, outsize);
606
- }
607
- assert(btype == 1 || btype == 2);
608
-
609
- ZopfliInitLZ77Store(&store);
610
-
611
- s.options = options;
612
- s.blockstart = instart;
613
- s.blockend = inend;
614
- #ifdef ZOPFLI_LONGEST_MATCH_CACHE
615
- s.lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache));
616
- ZopfliInitCache(inend - instart, s.lmc);
617
- #endif
618
-
619
- if (btype == 2) {
620
- ZopfliLZ77Optimal(&s, in, instart, inend, &store);
621
- } else {
622
- assert (btype == 1);
623
- ZopfliLZ77OptimalFixed(&s, in, instart, inend, &store);
624
- }
625
-
626
- if (btype == 1) {
627
- /* If all blocks are fixed tree, splitting into separate blocks only
628
- increases the total size. Leave npoints at 0, this represents 1 block. */
629
- } else {
630
- ZopfliBlockSplitLZ77(options, store.litlens, store.dists, store.size,
631
- options->blocksplittingmax, &splitpoints, &npoints);
632
- }
633
-
634
- for (i = 0; i <= npoints; i++) {
635
- size_t start = i == 0 ? 0 : splitpoints[i - 1];
636
- size_t end = i == npoints ? store.size : splitpoints[i];
637
- AddLZ77Block(options, btype, i == npoints && final,
638
- store.litlens, store.dists, start, end, 0,
639
- bp, out, outsize);
640
- }
641
-
642
- #ifdef ZOPFLI_LONGEST_MATCH_CACHE
643
- ZopfliCleanCache(s.lmc);
644
- free(s.lmc);
645
- #endif
646
-
647
- ZopfliCleanLZ77Store(&store);
648
- }
649
-
650
- /*
651
- Deflate a part, to allow ZopfliDeflate() to use multiple master blocks if
652
- needed.
653
- It is possible to call this function multiple times in a row, shifting
654
- instart and inend to next bytes of the data. If instart is larger than 0, then
655
- previous bytes are used as the initial dictionary for LZ77.
656
- This function will usually output multiple deflate blocks. If final is 1, then
657
- the final bit will be set on the last block.
658
- */
659
- void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
660
- const unsigned char* in, size_t instart, size_t inend,
661
- unsigned char* bp, unsigned char** out,
662
- size_t* outsize) {
663
- if (options->blocksplitting) {
664
- if (options->blocksplittinglast) {
665
- DeflateSplittingLast(options, btype, final, in, instart, inend,
666
- bp, out, outsize);
667
- } else {
668
- DeflateSplittingFirst(options, btype, final, in, instart, inend,
669
- bp, out, outsize);
670
- }
671
- } else {
672
- DeflateBlock(options, btype, final, in, instart, inend, bp, out, outsize);
673
- }
674
- }
675
-
676
- void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
677
- const unsigned char* in, size_t insize,
678
- unsigned char* bp, unsigned char** out, size_t* outsize) {
679
- #if ZOPFLI_MASTER_BLOCK_SIZE == 0
680
- ZopfliDeflatePart(options, btype, final, in, 0, insize, bp, out, outsize);
681
- #else
682
- size_t i = 0;
683
- while (i < insize) {
684
- int masterfinal = (i + ZOPFLI_MASTER_BLOCK_SIZE >= insize);
685
- int final2 = final && masterfinal;
686
- size_t size = masterfinal ? insize - i : ZOPFLI_MASTER_BLOCK_SIZE;
687
- ZopfliDeflatePart(options, btype, final2,
688
- in, i, i + size, bp, out, outsize);
689
- i += size;
690
- }
691
- #endif
692
- if (options->verbose) {
693
- fprintf(stderr,
694
- "Original Size: %d, Deflate: %d, Compression: %f%% Removed\n",
695
- (int)insize, (int)*outsize,
696
- 100.0 * (double)(insize - *outsize) / (double)insize);
697
- }
698
- }