zopfli 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +17 -0
- data/.gitmodules +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +34 -0
- data/Rakefile +25 -0
- data/ext/extconf.rb +43 -0
- data/ext/zopfli.c +38 -0
- data/lib/zopfli/version.rb +3 -0
- data/test/fixtures/alice29.txt +3609 -0
- data/test/test_zopfli_deflate.rb +24 -0
- data/vendor/zopfli/CONTRIBUTORS +6 -0
- data/vendor/zopfli/COPYING +201 -0
- data/vendor/zopfli/README +24 -0
- data/vendor/zopfli/blocksplitter.c +344 -0
- data/vendor/zopfli/blocksplitter.h +77 -0
- data/vendor/zopfli/cache.c +119 -0
- data/vendor/zopfli/cache.h +66 -0
- data/vendor/zopfli/deflate.c +697 -0
- data/vendor/zopfli/deflate.h +77 -0
- data/vendor/zopfli/gzip_container.c +117 -0
- data/vendor/zopfli/gzip_container.h +42 -0
- data/vendor/zopfli/hash.c +135 -0
- data/vendor/zopfli/hash.h +70 -0
- data/vendor/zopfli/katajainen.c +251 -0
- data/vendor/zopfli/katajainen.h +42 -0
- data/vendor/zopfli/lz77.c +468 -0
- data/vendor/zopfli/lz77.h +126 -0
- data/vendor/zopfli/makefile +5 -0
- data/vendor/zopfli/squeeze.c +543 -0
- data/vendor/zopfli/squeeze.h +60 -0
- data/vendor/zopfli/tree.c +101 -0
- data/vendor/zopfli/tree.h +51 -0
- data/vendor/zopfli/util.c +202 -0
- data/vendor/zopfli/util.h +175 -0
- data/vendor/zopfli/zlib_container.c +79 -0
- data/vendor/zopfli/zlib_container.h +42 -0
- data/vendor/zopfli/zopfli.h +71 -0
- data/vendor/zopfli/zopfli_bin.c +204 -0
- data/vendor/zopfli/zopfli_lib.c +37 -0
- data/zopfli.gemspec +49 -0
- metadata +130 -0
@@ -0,0 +1,77 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef ZOPFLI_DEFLATE_H_
|
21
|
+
#define ZOPFLI_DEFLATE_H_
|
22
|
+
|
23
|
+
/*
|
24
|
+
Functions to compress according to the DEFLATE specification, using the
|
25
|
+
"squeeze" LZ77 compression backend.
|
26
|
+
*/
|
27
|
+
|
28
|
+
#include "zopfli.h"
|
29
|
+
|
30
|
+
/*
|
31
|
+
Compresses according to the deflate specification and append the compressed
|
32
|
+
result to the output.
|
33
|
+
This function will usually output multiple deflate blocks. If final is 1, then
|
34
|
+
the final bit will be set on the last block.
|
35
|
+
|
36
|
+
options: global program options
|
37
|
+
btype: the deflate block type. Use 2 for best compression.
|
38
|
+
-0: non compressed blocks (00)
|
39
|
+
-1: blocks with fixed tree (01)
|
40
|
+
-2: blocks with dynamic tree (10)
|
41
|
+
final: whether this is the last section of the input, sets the final bit to the
|
42
|
+
last deflate block.
|
43
|
+
in: the input bytes
|
44
|
+
insize: number of input bytes
|
45
|
+
bp: bit pointer for the output array. This must initially be 0, and for
|
46
|
+
consecutive calls must be reused (it can have values from 0-7). This is
|
47
|
+
because deflate appends blocks as bit-based data, rather than on byte
|
48
|
+
boundaries.
|
49
|
+
out: pointer to the dynamic output array to which the result is appended. Must
|
50
|
+
be freed after use.
|
51
|
+
outsize: pointer to the dynamic output array size.
|
52
|
+
*/
|
53
|
+
void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
|
54
|
+
const unsigned char* in, size_t insize,
|
55
|
+
unsigned char* bp, unsigned char** out, size_t* outsize);
|
56
|
+
|
57
|
+
/*
|
58
|
+
Like ZopfliDeflate, but allows to specify start and end byte with instart and
|
59
|
+
inend. Only that part is compressed, but earlier bytes are still used for the
|
60
|
+
back window.
|
61
|
+
*/
|
62
|
+
void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
|
63
|
+
const unsigned char* in, size_t instart, size_t inend,
|
64
|
+
unsigned char* bp, unsigned char** out,
|
65
|
+
size_t* outsize);
|
66
|
+
|
67
|
+
/*
|
68
|
+
Calculates block size in bits.
|
69
|
+
litlens: lz77 lit/lengths
|
70
|
+
dists: ll77 distances
|
71
|
+
lstart: start of block
|
72
|
+
lend: end of block (not inclusive)
|
73
|
+
*/
|
74
|
+
double ZopfliCalculateBlockSize(const unsigned short* litlens,
|
75
|
+
const unsigned short* dists,
|
76
|
+
size_t lstart, size_t lend, int btype);
|
77
|
+
#endif /* ZOPFLI_DEFLATE_H_ */
|
@@ -0,0 +1,117 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2013 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include "gzip_container.h"
|
21
|
+
#include "util.h"
|
22
|
+
|
23
|
+
#include <stdio.h>
|
24
|
+
|
25
|
+
#include "deflate.h"
|
26
|
+
|
27
|
+
/* Table of CRCs of all 8-bit messages. */
|
28
|
+
static unsigned long crc_table[256];
|
29
|
+
|
30
|
+
/* Flag: has the table been computed? Initially false. */
|
31
|
+
static int crc_table_computed = 0;
|
32
|
+
|
33
|
+
/* Makes the table for a fast CRC. */
|
34
|
+
static void MakeCRCTable() {
|
35
|
+
unsigned long c;
|
36
|
+
int n, k;
|
37
|
+
for (n = 0; n < 256; n++) {
|
38
|
+
c = (unsigned long) n;
|
39
|
+
for (k = 0; k < 8; k++) {
|
40
|
+
if (c & 1) {
|
41
|
+
c = 0xedb88320L ^ (c >> 1);
|
42
|
+
} else {
|
43
|
+
c = c >> 1;
|
44
|
+
}
|
45
|
+
}
|
46
|
+
crc_table[n] = c;
|
47
|
+
}
|
48
|
+
crc_table_computed = 1;
|
49
|
+
}
|
50
|
+
|
51
|
+
|
52
|
+
/*
|
53
|
+
Updates a running crc with the bytes buf[0..len-1] and returns
|
54
|
+
the updated crc. The crc should be initialized to zero.
|
55
|
+
*/
|
56
|
+
static unsigned long UpdateCRC(unsigned long crc,
|
57
|
+
const unsigned char *buf, size_t len) {
|
58
|
+
unsigned long c = crc ^ 0xffffffffL;
|
59
|
+
unsigned n;
|
60
|
+
|
61
|
+
if (!crc_table_computed)
|
62
|
+
MakeCRCTable();
|
63
|
+
for (n = 0; n < len; n++) {
|
64
|
+
c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
|
65
|
+
}
|
66
|
+
return c ^ 0xffffffffL;
|
67
|
+
}
|
68
|
+
|
69
|
+
/* Returns the CRC of the bytes buf[0..len-1]. */
|
70
|
+
static unsigned long CRC(const unsigned char* buf, int len) {
|
71
|
+
return UpdateCRC(0L, buf, len);
|
72
|
+
}
|
73
|
+
|
74
|
+
/*
|
75
|
+
Compresses the data according to the gzip specification.
|
76
|
+
*/
|
77
|
+
void ZopfliGzipCompress(const ZopfliOptions* options,
|
78
|
+
const unsigned char* in, size_t insize,
|
79
|
+
unsigned char** out, size_t* outsize) {
|
80
|
+
unsigned long crcvalue = CRC(in, insize);
|
81
|
+
unsigned char bp = 0;
|
82
|
+
|
83
|
+
ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */
|
84
|
+
ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */
|
85
|
+
ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */
|
86
|
+
ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */
|
87
|
+
/* MTIME */
|
88
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
89
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
90
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
91
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
92
|
+
|
93
|
+
ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */
|
94
|
+
ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */
|
95
|
+
|
96
|
+
ZopfliDeflate(options, 2 /* Dynamic block */, 1,
|
97
|
+
in, insize, &bp, out, outsize);
|
98
|
+
|
99
|
+
/* CRC */
|
100
|
+
ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize);
|
101
|
+
ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize);
|
102
|
+
ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize);
|
103
|
+
ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize);
|
104
|
+
|
105
|
+
/* ISIZE */
|
106
|
+
ZOPFLI_APPEND_DATA(insize % 256, out, outsize);
|
107
|
+
ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize);
|
108
|
+
ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize);
|
109
|
+
ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize);
|
110
|
+
|
111
|
+
if (options->verbose) {
|
112
|
+
fprintf(stderr,
|
113
|
+
"Original Size: %d, Compressed: %d, Compression: %f%% Removed\n",
|
114
|
+
(int)insize, (int)*outsize,
|
115
|
+
100.0f * (float)(insize - *outsize) / (float)insize);
|
116
|
+
}
|
117
|
+
}
|
@@ -0,0 +1,42 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2013 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef ZOPFLI_GZIP_H_
|
21
|
+
#define ZOPFLI_GZIP_H_
|
22
|
+
|
23
|
+
/*
|
24
|
+
Functions to compress according to the Gzip specification.
|
25
|
+
*/
|
26
|
+
|
27
|
+
#include "zopfli.h"
|
28
|
+
|
29
|
+
/*
|
30
|
+
Compresses according to the gzip specification and append the compressed
|
31
|
+
result to the output.
|
32
|
+
|
33
|
+
options: global program options
|
34
|
+
out: pointer to the dynamic output array to which the result is appended. Must
|
35
|
+
be freed after use.
|
36
|
+
outsize: pointer to the dynamic output array size.
|
37
|
+
*/
|
38
|
+
void ZopfliGzipCompress(const ZopfliOptions* options,
|
39
|
+
const unsigned char* in, size_t insize,
|
40
|
+
unsigned char** out, size_t* outsize);
|
41
|
+
|
42
|
+
#endif /* ZOPFLI_GZIP_H_ */
|
@@ -0,0 +1,135 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include "hash.h"
|
21
|
+
|
22
|
+
#include <assert.h>
|
23
|
+
#include <stdio.h>
|
24
|
+
#include <stdlib.h>
|
25
|
+
|
26
|
+
#define HASH_SHIFT 5
|
27
|
+
#define HASH_MASK 32767
|
28
|
+
|
29
|
+
void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
|
30
|
+
size_t i;
|
31
|
+
|
32
|
+
h->val = 0;
|
33
|
+
h->head = (int*)malloc(sizeof(*h->head) * 65536);
|
34
|
+
h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size);
|
35
|
+
h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size);
|
36
|
+
for (i = 0; i < 65536; i++) {
|
37
|
+
h->head[i] = -1; /* -1 indicates no head so far. */
|
38
|
+
}
|
39
|
+
for (i = 0; i < window_size; i++) {
|
40
|
+
h->prev[i] = i; /* If prev[j] == j, then prev[j] is uninitialized. */
|
41
|
+
h->hashval[i] = -1;
|
42
|
+
}
|
43
|
+
|
44
|
+
#ifdef ZOPFLI_HASH_SAME
|
45
|
+
h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
|
46
|
+
for (i = 0; i < window_size; i++) {
|
47
|
+
h->same[i] = 0;
|
48
|
+
}
|
49
|
+
#endif
|
50
|
+
|
51
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
52
|
+
h->val2 = 0;
|
53
|
+
h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
|
54
|
+
h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
|
55
|
+
h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
|
56
|
+
for (i = 0; i < 65536; i++) {
|
57
|
+
h->head2[i] = -1;
|
58
|
+
}
|
59
|
+
for (i = 0; i < window_size; i++) {
|
60
|
+
h->prev2[i] = i;
|
61
|
+
h->hashval2[i] = -1;
|
62
|
+
}
|
63
|
+
#endif
|
64
|
+
}
|
65
|
+
|
66
|
+
void ZopfliCleanHash(ZopfliHash* h) {
|
67
|
+
free(h->head);
|
68
|
+
free(h->prev);
|
69
|
+
free(h->hashval);
|
70
|
+
|
71
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
72
|
+
free(h->head2);
|
73
|
+
free(h->prev2);
|
74
|
+
free(h->hashval2);
|
75
|
+
#endif
|
76
|
+
|
77
|
+
#ifdef ZOPFLI_HASH_SAME
|
78
|
+
free(h->same);
|
79
|
+
#endif
|
80
|
+
}
|
81
|
+
|
82
|
+
/*
|
83
|
+
Update the sliding hash value with the given byte. All calls to this function
|
84
|
+
must be made on consecutive input characters. Since the hash value exists out
|
85
|
+
of multiple input bytes, a few warmups with this function are needed initially.
|
86
|
+
*/
|
87
|
+
static void UpdateHashValue(ZopfliHash* h, unsigned char c) {
|
88
|
+
h->val = (((h->val) << HASH_SHIFT) ^ (c)) & HASH_MASK;
|
89
|
+
}
|
90
|
+
|
91
|
+
void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
|
92
|
+
ZopfliHash* h) {
|
93
|
+
unsigned short hpos = pos & ZOPFLI_WINDOW_MASK;
|
94
|
+
#ifdef ZOPFLI_HASH_SAME
|
95
|
+
size_t amount = 0;
|
96
|
+
#endif
|
97
|
+
|
98
|
+
UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ?
|
99
|
+
array[pos + ZOPFLI_MIN_MATCH - 1] : 0);
|
100
|
+
h->hashval[hpos] = h->val;
|
101
|
+
if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) {
|
102
|
+
h->prev[hpos] = h->head[h->val];
|
103
|
+
}
|
104
|
+
else h->prev[hpos] = hpos;
|
105
|
+
h->head[h->val] = hpos;
|
106
|
+
|
107
|
+
#ifdef ZOPFLI_HASH_SAME
|
108
|
+
/* Update "same". */
|
109
|
+
if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) {
|
110
|
+
amount = h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] - 1;
|
111
|
+
}
|
112
|
+
while (pos + amount + 1 < end &&
|
113
|
+
array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) {
|
114
|
+
amount++;
|
115
|
+
}
|
116
|
+
h->same[hpos] = amount;
|
117
|
+
#endif
|
118
|
+
|
119
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
120
|
+
h->val2 = ((h->same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h->val;
|
121
|
+
h->hashval2[hpos] = h->val2;
|
122
|
+
if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) {
|
123
|
+
h->prev2[hpos] = h->head2[h->val2];
|
124
|
+
}
|
125
|
+
else h->prev2[hpos] = hpos;
|
126
|
+
h->head2[h->val2] = hpos;
|
127
|
+
#endif
|
128
|
+
}
|
129
|
+
|
130
|
+
void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
|
131
|
+
ZopfliHash* h) {
|
132
|
+
(void)end;
|
133
|
+
UpdateHashValue(h, array[pos + 0]);
|
134
|
+
UpdateHashValue(h, array[pos + 1]);
|
135
|
+
}
|
@@ -0,0 +1,70 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
/*
|
21
|
+
The hash for ZopfliFindLongestMatch of lz77.c.
|
22
|
+
*/
|
23
|
+
|
24
|
+
#ifndef ZOPFLI_HASH_H_
|
25
|
+
#define ZOPFLI_HASH_H_
|
26
|
+
|
27
|
+
#include "util.h"
|
28
|
+
|
29
|
+
typedef struct ZopfliHash {
|
30
|
+
int* head; /* Hash value to index of its most recent occurance. */
|
31
|
+
unsigned short* prev; /* Index to index of prev. occurance of same hash. */
|
32
|
+
int* hashval; /* Index to hash value at this index. */
|
33
|
+
int val; /* Current hash value. */
|
34
|
+
|
35
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
36
|
+
/* Fields with similar purpose as the above hash, but for the second hash with
|
37
|
+
a value that is calculated differently. */
|
38
|
+
int* head2; /* Hash value to index of its most recent occurance. */
|
39
|
+
unsigned short* prev2; /* Index to index of prev. occurance of same hash. */
|
40
|
+
int* hashval2; /* Index to hash value at this index. */
|
41
|
+
int val2; /* Current hash value. */
|
42
|
+
#endif
|
43
|
+
|
44
|
+
#ifdef ZOPFLI_HASH_SAME
|
45
|
+
unsigned short* same; /* Amount of repetitions of same byte after this .*/
|
46
|
+
#endif
|
47
|
+
} ZopfliHash;
|
48
|
+
|
49
|
+
/* Allocates and initializes all fields of ZopfliHash. */
|
50
|
+
void ZopfliInitHash(size_t window_size, ZopfliHash* h);
|
51
|
+
|
52
|
+
/* Frees all fields of ZopfliHash. */
|
53
|
+
void ZopfliCleanHash(ZopfliHash* h);
|
54
|
+
|
55
|
+
/*
|
56
|
+
Updates the hash values based on the current position in the array. All calls
|
57
|
+
to this must be made for consecutive bytes.
|
58
|
+
*/
|
59
|
+
void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
|
60
|
+
ZopfliHash* h);
|
61
|
+
|
62
|
+
/*
|
63
|
+
Prepopulates hash:
|
64
|
+
Fills in the initial values in the hash, before ZopfliUpdateHash can be used
|
65
|
+
correctly.
|
66
|
+
*/
|
67
|
+
void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
|
68
|
+
ZopfliHash* h);
|
69
|
+
|
70
|
+
#endif /* ZOPFLI_HASH_H_ */
|
@@ -0,0 +1,251 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
/*
|
21
|
+
Bounded package merge algorithm, based on the paper
|
22
|
+
"A Fast and Space-Economical Algorithm for Length-Limited Coding
|
23
|
+
Jyrki Katajainen, Alistair Moffat, Andrew Turpin".
|
24
|
+
*/
|
25
|
+
|
26
|
+
#include "katajainen.h"
|
27
|
+
#include <assert.h>
|
28
|
+
#include <stdlib.h>
|
29
|
+
|
30
|
+
typedef struct Node Node;
|
31
|
+
|
32
|
+
/*
|
33
|
+
Nodes forming chains. Also used to represent leaves.
|
34
|
+
*/
|
35
|
+
struct Node {
|
36
|
+
size_t weight; /* Total weight (symbol count) of this chain. */
|
37
|
+
Node* tail; /* Previous node(s) of this chain, or 0 if none. */
|
38
|
+
int count; /* Leaf symbol index, or number of leaves before this chain. */
|
39
|
+
char inuse; /* Tracking for garbage collection. */
|
40
|
+
};
|
41
|
+
|
42
|
+
/*
|
43
|
+
Memory pool for nodes.
|
44
|
+
*/
|
45
|
+
typedef struct NodePool {
|
46
|
+
Node* nodes; /* The pool. */
|
47
|
+
Node* next; /* Pointer to a possibly free node in the pool. */
|
48
|
+
int size; /* Size of the memory pool. */
|
49
|
+
} NodePool;
|
50
|
+
|
51
|
+
/*
|
52
|
+
Initializes a chain node with the given values and marks it as in use.
|
53
|
+
*/
|
54
|
+
static void InitNode(size_t weight, int count, Node* tail, Node* node) {
|
55
|
+
node->weight = weight;
|
56
|
+
node->count = count;
|
57
|
+
node->tail = tail;
|
58
|
+
node->inuse = 1;
|
59
|
+
}
|
60
|
+
|
61
|
+
/*
|
62
|
+
Finds a free location in the memory pool. Performs garbage collection if needed.
|
63
|
+
lists: If given, used to mark in-use nodes during garbage collection.
|
64
|
+
maxbits: Size of lists.
|
65
|
+
pool: Memory pool to get free node from.
|
66
|
+
*/
|
67
|
+
static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) {
|
68
|
+
for (;;) {
|
69
|
+
if (pool->next >= &pool->nodes[pool->size]) {
|
70
|
+
/* Garbage collection. */
|
71
|
+
int i;
|
72
|
+
for (i = 0; i < pool->size; i++) {
|
73
|
+
pool->nodes[i].inuse = 0;
|
74
|
+
}
|
75
|
+
if (lists) {
|
76
|
+
for (i = 0; i < maxbits * 2; i++) {
|
77
|
+
Node* node;
|
78
|
+
for (node = lists[i / 2][i % 2]; node; node = node->tail) {
|
79
|
+
node->inuse = 1;
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
pool->next = &pool->nodes[0];
|
84
|
+
}
|
85
|
+
if (!pool->next->inuse) break; /* Found one. */
|
86
|
+
pool->next++;
|
87
|
+
}
|
88
|
+
return pool->next++;
|
89
|
+
}
|
90
|
+
|
91
|
+
|
92
|
+
/*
|
93
|
+
Performs a Boundary Package-Merge step. Puts a new chain in the given list. The
|
94
|
+
new chain is, depending on the weights, a leaf or a combination of two chains
|
95
|
+
from the previous list.
|
96
|
+
lists: The lists of chains.
|
97
|
+
maxbits: Number of lists.
|
98
|
+
leaves: The leaves, one per symbol.
|
99
|
+
numsymbols: Number of leaves.
|
100
|
+
pool: the node memory pool.
|
101
|
+
index: The index of the list in which a new chain or leaf is required.
|
102
|
+
final: Whether this is the last time this function is called. If it is then it
|
103
|
+
is no more needed to recursively call self.
|
104
|
+
*/
|
105
|
+
static void BoundaryPM(Node* (*lists)[2], int maxbits,
|
106
|
+
Node* leaves, int numsymbols, NodePool* pool, int index, char final) {
|
107
|
+
Node* newchain;
|
108
|
+
Node* oldchain;
|
109
|
+
int lastcount = lists[index][1]->count; /* Count of last chain of list. */
|
110
|
+
|
111
|
+
if (index == 0 && lastcount >= numsymbols) return;
|
112
|
+
|
113
|
+
newchain = GetFreeNode(lists, maxbits, pool);
|
114
|
+
oldchain = lists[index][1];
|
115
|
+
|
116
|
+
/* These are set up before the recursive calls below, so that there is a list
|
117
|
+
pointing to the new node, to let the garbage collection know it's in use. */
|
118
|
+
lists[index][0] = oldchain;
|
119
|
+
lists[index][1] = newchain;
|
120
|
+
|
121
|
+
if (index == 0) {
|
122
|
+
/* New leaf node in list 0. */
|
123
|
+
InitNode(leaves[lastcount].weight, lastcount + 1, 0, newchain);
|
124
|
+
} else {
|
125
|
+
size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
|
126
|
+
if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
|
127
|
+
/* New leaf inserted in list, so count is incremented. */
|
128
|
+
InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail,
|
129
|
+
newchain);
|
130
|
+
} else {
|
131
|
+
InitNode(sum, lastcount, lists[index - 1][1], newchain);
|
132
|
+
if (!final) {
|
133
|
+
/* Two lookahead chains of previous list used up, create new ones. */
|
134
|
+
BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
|
135
|
+
BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
|
136
|
+
}
|
137
|
+
}
|
138
|
+
}
|
139
|
+
}
|
140
|
+
|
141
|
+
/*
|
142
|
+
Initializes each list with as lookahead chains the two leaves with lowest
|
143
|
+
weights.
|
144
|
+
*/
|
145
|
+
static void InitLists(
|
146
|
+
NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) {
|
147
|
+
int i;
|
148
|
+
Node* node0 = GetFreeNode(0, maxbits, pool);
|
149
|
+
Node* node1 = GetFreeNode(0, maxbits, pool);
|
150
|
+
InitNode(leaves[0].weight, 1, 0, node0);
|
151
|
+
InitNode(leaves[1].weight, 2, 0, node1);
|
152
|
+
for (i = 0; i < maxbits; i++) {
|
153
|
+
lists[i][0] = node0;
|
154
|
+
lists[i][1] = node1;
|
155
|
+
}
|
156
|
+
}
|
157
|
+
|
158
|
+
/*
|
159
|
+
Converts result of boundary package-merge to the bitlengths. The result in the
|
160
|
+
last chain of the last list contains the amount of active leaves in each list.
|
161
|
+
chain: Chain to extract the bit length from (last chain from last list).
|
162
|
+
*/
|
163
|
+
static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) {
|
164
|
+
Node* node;
|
165
|
+
for (node = chain; node; node = node->tail) {
|
166
|
+
int i;
|
167
|
+
for (i = 0; i < node->count; i++) {
|
168
|
+
bitlengths[leaves[i].count]++;
|
169
|
+
}
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
/*
|
174
|
+
Comparator for sorting the leaves. Has the function signature for qsort.
|
175
|
+
*/
|
176
|
+
static int LeafComparator(const void* a, const void* b) {
|
177
|
+
return ((const Node*)a)->weight - ((const Node*)b)->weight;
|
178
|
+
}
|
179
|
+
|
180
|
+
int ZopfliLengthLimitedCodeLengths(
|
181
|
+
const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) {
|
182
|
+
NodePool pool;
|
183
|
+
int i;
|
184
|
+
int numsymbols = 0; /* Amount of symbols with frequency > 0. */
|
185
|
+
int numBoundaryPMRuns;
|
186
|
+
|
187
|
+
/* Array of lists of chains. Each list requires only two lookahead chains at
|
188
|
+
a time, so each list is a array of two Node*'s. */
|
189
|
+
Node* (*lists)[2];
|
190
|
+
|
191
|
+
/* One leaf per symbol. Only numsymbols leaves will be used. */
|
192
|
+
Node* leaves = (Node*)malloc(n * sizeof(*leaves));
|
193
|
+
|
194
|
+
/* Initialize all bitlengths at 0. */
|
195
|
+
for (i = 0; i < n; i++) {
|
196
|
+
bitlengths[i] = 0;
|
197
|
+
}
|
198
|
+
|
199
|
+
/* Count used symbols and place them in the leaves. */
|
200
|
+
for (i = 0; i < n; i++) {
|
201
|
+
if (frequencies[i]) {
|
202
|
+
leaves[numsymbols].weight = frequencies[i];
|
203
|
+
leaves[numsymbols].count = i; /* Index of symbol this leaf represents. */
|
204
|
+
numsymbols++;
|
205
|
+
}
|
206
|
+
}
|
207
|
+
|
208
|
+
/* Check special cases and error conditions. */
|
209
|
+
if ((1 << maxbits) < numsymbols) {
|
210
|
+
free(leaves);
|
211
|
+
return 1; /* Error, too few maxbits to represent symbols. */
|
212
|
+
}
|
213
|
+
if (numsymbols == 0) {
|
214
|
+
free(leaves);
|
215
|
+
return 0; /* No symbols at all. OK. */
|
216
|
+
}
|
217
|
+
if (numsymbols == 1) {
|
218
|
+
bitlengths[leaves[0].count] = 1;
|
219
|
+
free(leaves);
|
220
|
+
return 0; /* Only one symbol, give it bitlength 1, not 0. OK. */
|
221
|
+
}
|
222
|
+
|
223
|
+
/* Sort the leaves from lightest to heaviest. */
|
224
|
+
qsort(leaves, numsymbols, sizeof(Node), LeafComparator);
|
225
|
+
|
226
|
+
/* Initialize node memory pool. */
|
227
|
+
pool.size = 2 * maxbits * (maxbits + 1);
|
228
|
+
pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes));
|
229
|
+
pool.next = pool.nodes;
|
230
|
+
for (i = 0; i < pool.size; i++) {
|
231
|
+
pool.nodes[i].inuse = 0;
|
232
|
+
}
|
233
|
+
|
234
|
+
lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists));
|
235
|
+
InitLists(&pool, leaves, maxbits, lists);
|
236
|
+
|
237
|
+
/* In the last list, 2 * numsymbols - 2 active chains need to be created. Two
|
238
|
+
are already created in the initialization. Each BoundaryPM run creates one. */
|
239
|
+
numBoundaryPMRuns = 2 * numsymbols - 4;
|
240
|
+
for (i = 0; i < numBoundaryPMRuns; i++) {
|
241
|
+
char final = i == numBoundaryPMRuns - 1;
|
242
|
+
BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final);
|
243
|
+
}
|
244
|
+
|
245
|
+
ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths);
|
246
|
+
|
247
|
+
free(lists);
|
248
|
+
free(leaves);
|
249
|
+
free(pool.nodes);
|
250
|
+
return 0; /* OK. */
|
251
|
+
}
|