zopfli 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +17 -0
- data/.gitmodules +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +34 -0
- data/Rakefile +25 -0
- data/ext/extconf.rb +43 -0
- data/ext/zopfli.c +38 -0
- data/lib/zopfli/version.rb +3 -0
- data/test/fixtures/alice29.txt +3609 -0
- data/test/test_zopfli_deflate.rb +24 -0
- data/vendor/zopfli/CONTRIBUTORS +6 -0
- data/vendor/zopfli/COPYING +201 -0
- data/vendor/zopfli/README +24 -0
- data/vendor/zopfli/blocksplitter.c +344 -0
- data/vendor/zopfli/blocksplitter.h +77 -0
- data/vendor/zopfli/cache.c +119 -0
- data/vendor/zopfli/cache.h +66 -0
- data/vendor/zopfli/deflate.c +697 -0
- data/vendor/zopfli/deflate.h +77 -0
- data/vendor/zopfli/gzip_container.c +117 -0
- data/vendor/zopfli/gzip_container.h +42 -0
- data/vendor/zopfli/hash.c +135 -0
- data/vendor/zopfli/hash.h +70 -0
- data/vendor/zopfli/katajainen.c +251 -0
- data/vendor/zopfli/katajainen.h +42 -0
- data/vendor/zopfli/lz77.c +468 -0
- data/vendor/zopfli/lz77.h +126 -0
- data/vendor/zopfli/makefile +5 -0
- data/vendor/zopfli/squeeze.c +543 -0
- data/vendor/zopfli/squeeze.h +60 -0
- data/vendor/zopfli/tree.c +101 -0
- data/vendor/zopfli/tree.h +51 -0
- data/vendor/zopfli/util.c +202 -0
- data/vendor/zopfli/util.h +175 -0
- data/vendor/zopfli/zlib_container.c +79 -0
- data/vendor/zopfli/zlib_container.h +42 -0
- data/vendor/zopfli/zopfli.h +71 -0
- data/vendor/zopfli/zopfli_bin.c +204 -0
- data/vendor/zopfli/zopfli_lib.c +37 -0
- data/zopfli.gemspec +49 -0
- metadata +130 -0
@@ -0,0 +1,77 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef ZOPFLI_DEFLATE_H_
|
21
|
+
#define ZOPFLI_DEFLATE_H_
|
22
|
+
|
23
|
+
/*
|
24
|
+
Functions to compress according to the DEFLATE specification, using the
|
25
|
+
"squeeze" LZ77 compression backend.
|
26
|
+
*/
|
27
|
+
|
28
|
+
#include "zopfli.h"
|
29
|
+
|
30
|
+
/*
|
31
|
+
Compresses according to the deflate specification and append the compressed
|
32
|
+
result to the output.
|
33
|
+
This function will usually output multiple deflate blocks. If final is 1, then
|
34
|
+
the final bit will be set on the last block.
|
35
|
+
|
36
|
+
options: global program options
|
37
|
+
btype: the deflate block type. Use 2 for best compression.
|
38
|
+
-0: non compressed blocks (00)
|
39
|
+
-1: blocks with fixed tree (01)
|
40
|
+
-2: blocks with dynamic tree (10)
|
41
|
+
final: whether this is the last section of the input, sets the final bit to the
|
42
|
+
last deflate block.
|
43
|
+
in: the input bytes
|
44
|
+
insize: number of input bytes
|
45
|
+
bp: bit pointer for the output array. This must initially be 0, and for
|
46
|
+
consecutive calls must be reused (it can have values from 0-7). This is
|
47
|
+
because deflate appends blocks as bit-based data, rather than on byte
|
48
|
+
boundaries.
|
49
|
+
out: pointer to the dynamic output array to which the result is appended. Must
|
50
|
+
be freed after use.
|
51
|
+
outsize: pointer to the dynamic output array size.
|
52
|
+
*/
|
53
|
+
void ZopfliDeflate(const ZopfliOptions* options, int btype, int final,
|
54
|
+
const unsigned char* in, size_t insize,
|
55
|
+
unsigned char* bp, unsigned char** out, size_t* outsize);
|
56
|
+
|
57
|
+
/*
|
58
|
+
Like ZopfliDeflate, but allows to specify start and end byte with instart and
|
59
|
+
inend. Only that part is compressed, but earlier bytes are still used for the
|
60
|
+
back window.
|
61
|
+
*/
|
62
|
+
void ZopfliDeflatePart(const ZopfliOptions* options, int btype, int final,
|
63
|
+
const unsigned char* in, size_t instart, size_t inend,
|
64
|
+
unsigned char* bp, unsigned char** out,
|
65
|
+
size_t* outsize);
|
66
|
+
|
67
|
+
/*
|
68
|
+
Calculates block size in bits.
|
69
|
+
litlens: lz77 lit/lengths
|
70
|
+
dists: ll77 distances
|
71
|
+
lstart: start of block
|
72
|
+
lend: end of block (not inclusive)
|
73
|
+
*/
|
74
|
+
double ZopfliCalculateBlockSize(const unsigned short* litlens,
|
75
|
+
const unsigned short* dists,
|
76
|
+
size_t lstart, size_t lend, int btype);
|
77
|
+
#endif /* ZOPFLI_DEFLATE_H_ */
|
@@ -0,0 +1,117 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2013 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include "gzip_container.h"
|
21
|
+
#include "util.h"
|
22
|
+
|
23
|
+
#include <stdio.h>
|
24
|
+
|
25
|
+
#include "deflate.h"
|
26
|
+
|
27
|
+
/* Table of CRCs of all 8-bit messages. */
|
28
|
+
static unsigned long crc_table[256];
|
29
|
+
|
30
|
+
/* Flag: has the table been computed? Initially false. */
|
31
|
+
static int crc_table_computed = 0;
|
32
|
+
|
33
|
+
/* Makes the table for a fast CRC. */
|
34
|
+
static void MakeCRCTable() {
|
35
|
+
unsigned long c;
|
36
|
+
int n, k;
|
37
|
+
for (n = 0; n < 256; n++) {
|
38
|
+
c = (unsigned long) n;
|
39
|
+
for (k = 0; k < 8; k++) {
|
40
|
+
if (c & 1) {
|
41
|
+
c = 0xedb88320L ^ (c >> 1);
|
42
|
+
} else {
|
43
|
+
c = c >> 1;
|
44
|
+
}
|
45
|
+
}
|
46
|
+
crc_table[n] = c;
|
47
|
+
}
|
48
|
+
crc_table_computed = 1;
|
49
|
+
}
|
50
|
+
|
51
|
+
|
52
|
+
/*
|
53
|
+
Updates a running crc with the bytes buf[0..len-1] and returns
|
54
|
+
the updated crc. The crc should be initialized to zero.
|
55
|
+
*/
|
56
|
+
static unsigned long UpdateCRC(unsigned long crc,
|
57
|
+
const unsigned char *buf, size_t len) {
|
58
|
+
unsigned long c = crc ^ 0xffffffffL;
|
59
|
+
unsigned n;
|
60
|
+
|
61
|
+
if (!crc_table_computed)
|
62
|
+
MakeCRCTable();
|
63
|
+
for (n = 0; n < len; n++) {
|
64
|
+
c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
|
65
|
+
}
|
66
|
+
return c ^ 0xffffffffL;
|
67
|
+
}
|
68
|
+
|
69
|
+
/* Returns the CRC of the bytes buf[0..len-1]. */
|
70
|
+
static unsigned long CRC(const unsigned char* buf, int len) {
|
71
|
+
return UpdateCRC(0L, buf, len);
|
72
|
+
}
|
73
|
+
|
74
|
+
/*
|
75
|
+
Compresses the data according to the gzip specification.
|
76
|
+
*/
|
77
|
+
void ZopfliGzipCompress(const ZopfliOptions* options,
|
78
|
+
const unsigned char* in, size_t insize,
|
79
|
+
unsigned char** out, size_t* outsize) {
|
80
|
+
unsigned long crcvalue = CRC(in, insize);
|
81
|
+
unsigned char bp = 0;
|
82
|
+
|
83
|
+
ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */
|
84
|
+
ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */
|
85
|
+
ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */
|
86
|
+
ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */
|
87
|
+
/* MTIME */
|
88
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
89
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
90
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
91
|
+
ZOPFLI_APPEND_DATA(0, out, outsize);
|
92
|
+
|
93
|
+
ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */
|
94
|
+
ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */
|
95
|
+
|
96
|
+
ZopfliDeflate(options, 2 /* Dynamic block */, 1,
|
97
|
+
in, insize, &bp, out, outsize);
|
98
|
+
|
99
|
+
/* CRC */
|
100
|
+
ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize);
|
101
|
+
ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize);
|
102
|
+
ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize);
|
103
|
+
ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize);
|
104
|
+
|
105
|
+
/* ISIZE */
|
106
|
+
ZOPFLI_APPEND_DATA(insize % 256, out, outsize);
|
107
|
+
ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize);
|
108
|
+
ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize);
|
109
|
+
ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize);
|
110
|
+
|
111
|
+
if (options->verbose) {
|
112
|
+
fprintf(stderr,
|
113
|
+
"Original Size: %d, Compressed: %d, Compression: %f%% Removed\n",
|
114
|
+
(int)insize, (int)*outsize,
|
115
|
+
100.0f * (float)(insize - *outsize) / (float)insize);
|
116
|
+
}
|
117
|
+
}
|
@@ -0,0 +1,42 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2013 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#ifndef ZOPFLI_GZIP_H_
|
21
|
+
#define ZOPFLI_GZIP_H_
|
22
|
+
|
23
|
+
/*
|
24
|
+
Functions to compress according to the Gzip specification.
|
25
|
+
*/
|
26
|
+
|
27
|
+
#include "zopfli.h"
|
28
|
+
|
29
|
+
/*
|
30
|
+
Compresses according to the gzip specification and append the compressed
|
31
|
+
result to the output.
|
32
|
+
|
33
|
+
options: global program options
|
34
|
+
out: pointer to the dynamic output array to which the result is appended. Must
|
35
|
+
be freed after use.
|
36
|
+
outsize: pointer to the dynamic output array size.
|
37
|
+
*/
|
38
|
+
void ZopfliGzipCompress(const ZopfliOptions* options,
|
39
|
+
const unsigned char* in, size_t insize,
|
40
|
+
unsigned char** out, size_t* outsize);
|
41
|
+
|
42
|
+
#endif /* ZOPFLI_GZIP_H_ */
|
@@ -0,0 +1,135 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include "hash.h"
|
21
|
+
|
22
|
+
#include <assert.h>
|
23
|
+
#include <stdio.h>
|
24
|
+
#include <stdlib.h>
|
25
|
+
|
26
|
+
#define HASH_SHIFT 5
|
27
|
+
#define HASH_MASK 32767
|
28
|
+
|
29
|
+
void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
|
30
|
+
size_t i;
|
31
|
+
|
32
|
+
h->val = 0;
|
33
|
+
h->head = (int*)malloc(sizeof(*h->head) * 65536);
|
34
|
+
h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size);
|
35
|
+
h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size);
|
36
|
+
for (i = 0; i < 65536; i++) {
|
37
|
+
h->head[i] = -1; /* -1 indicates no head so far. */
|
38
|
+
}
|
39
|
+
for (i = 0; i < window_size; i++) {
|
40
|
+
h->prev[i] = i; /* If prev[j] == j, then prev[j] is uninitialized. */
|
41
|
+
h->hashval[i] = -1;
|
42
|
+
}
|
43
|
+
|
44
|
+
#ifdef ZOPFLI_HASH_SAME
|
45
|
+
h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
|
46
|
+
for (i = 0; i < window_size; i++) {
|
47
|
+
h->same[i] = 0;
|
48
|
+
}
|
49
|
+
#endif
|
50
|
+
|
51
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
52
|
+
h->val2 = 0;
|
53
|
+
h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
|
54
|
+
h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
|
55
|
+
h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
|
56
|
+
for (i = 0; i < 65536; i++) {
|
57
|
+
h->head2[i] = -1;
|
58
|
+
}
|
59
|
+
for (i = 0; i < window_size; i++) {
|
60
|
+
h->prev2[i] = i;
|
61
|
+
h->hashval2[i] = -1;
|
62
|
+
}
|
63
|
+
#endif
|
64
|
+
}
|
65
|
+
|
66
|
+
void ZopfliCleanHash(ZopfliHash* h) {
|
67
|
+
free(h->head);
|
68
|
+
free(h->prev);
|
69
|
+
free(h->hashval);
|
70
|
+
|
71
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
72
|
+
free(h->head2);
|
73
|
+
free(h->prev2);
|
74
|
+
free(h->hashval2);
|
75
|
+
#endif
|
76
|
+
|
77
|
+
#ifdef ZOPFLI_HASH_SAME
|
78
|
+
free(h->same);
|
79
|
+
#endif
|
80
|
+
}
|
81
|
+
|
82
|
+
/*
|
83
|
+
Update the sliding hash value with the given byte. All calls to this function
|
84
|
+
must be made on consecutive input characters. Since the hash value exists out
|
85
|
+
of multiple input bytes, a few warmups with this function are needed initially.
|
86
|
+
*/
|
87
|
+
static void UpdateHashValue(ZopfliHash* h, unsigned char c) {
|
88
|
+
h->val = (((h->val) << HASH_SHIFT) ^ (c)) & HASH_MASK;
|
89
|
+
}
|
90
|
+
|
91
|
+
void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
|
92
|
+
ZopfliHash* h) {
|
93
|
+
unsigned short hpos = pos & ZOPFLI_WINDOW_MASK;
|
94
|
+
#ifdef ZOPFLI_HASH_SAME
|
95
|
+
size_t amount = 0;
|
96
|
+
#endif
|
97
|
+
|
98
|
+
UpdateHashValue(h, pos + ZOPFLI_MIN_MATCH <= end ?
|
99
|
+
array[pos + ZOPFLI_MIN_MATCH - 1] : 0);
|
100
|
+
h->hashval[hpos] = h->val;
|
101
|
+
if (h->head[h->val] != -1 && h->hashval[h->head[h->val]] == h->val) {
|
102
|
+
h->prev[hpos] = h->head[h->val];
|
103
|
+
}
|
104
|
+
else h->prev[hpos] = hpos;
|
105
|
+
h->head[h->val] = hpos;
|
106
|
+
|
107
|
+
#ifdef ZOPFLI_HASH_SAME
|
108
|
+
/* Update "same". */
|
109
|
+
if (h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] > 1) {
|
110
|
+
amount = h->same[(pos - 1) & ZOPFLI_WINDOW_MASK] - 1;
|
111
|
+
}
|
112
|
+
while (pos + amount + 1 < end &&
|
113
|
+
array[pos] == array[pos + amount + 1] && amount < (unsigned short)(-1)) {
|
114
|
+
amount++;
|
115
|
+
}
|
116
|
+
h->same[hpos] = amount;
|
117
|
+
#endif
|
118
|
+
|
119
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
120
|
+
h->val2 = ((h->same[hpos] - ZOPFLI_MIN_MATCH) & 255) ^ h->val;
|
121
|
+
h->hashval2[hpos] = h->val2;
|
122
|
+
if (h->head2[h->val2] != -1 && h->hashval2[h->head2[h->val2]] == h->val2) {
|
123
|
+
h->prev2[hpos] = h->head2[h->val2];
|
124
|
+
}
|
125
|
+
else h->prev2[hpos] = hpos;
|
126
|
+
h->head2[h->val2] = hpos;
|
127
|
+
#endif
|
128
|
+
}
|
129
|
+
|
130
|
+
void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
|
131
|
+
ZopfliHash* h) {
|
132
|
+
(void)end;
|
133
|
+
UpdateHashValue(h, array[pos + 0]);
|
134
|
+
UpdateHashValue(h, array[pos + 1]);
|
135
|
+
}
|
@@ -0,0 +1,70 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
/*
|
21
|
+
The hash for ZopfliFindLongestMatch of lz77.c.
|
22
|
+
*/
|
23
|
+
|
24
|
+
#ifndef ZOPFLI_HASH_H_
|
25
|
+
#define ZOPFLI_HASH_H_
|
26
|
+
|
27
|
+
#include "util.h"
|
28
|
+
|
29
|
+
typedef struct ZopfliHash {
|
30
|
+
int* head; /* Hash value to index of its most recent occurance. */
|
31
|
+
unsigned short* prev; /* Index to index of prev. occurance of same hash. */
|
32
|
+
int* hashval; /* Index to hash value at this index. */
|
33
|
+
int val; /* Current hash value. */
|
34
|
+
|
35
|
+
#ifdef ZOPFLI_HASH_SAME_HASH
|
36
|
+
/* Fields with similar purpose as the above hash, but for the second hash with
|
37
|
+
a value that is calculated differently. */
|
38
|
+
int* head2; /* Hash value to index of its most recent occurance. */
|
39
|
+
unsigned short* prev2; /* Index to index of prev. occurance of same hash. */
|
40
|
+
int* hashval2; /* Index to hash value at this index. */
|
41
|
+
int val2; /* Current hash value. */
|
42
|
+
#endif
|
43
|
+
|
44
|
+
#ifdef ZOPFLI_HASH_SAME
|
45
|
+
unsigned short* same; /* Amount of repetitions of same byte after this .*/
|
46
|
+
#endif
|
47
|
+
} ZopfliHash;
|
48
|
+
|
49
|
+
/* Allocates and initializes all fields of ZopfliHash. */
|
50
|
+
void ZopfliInitHash(size_t window_size, ZopfliHash* h);
|
51
|
+
|
52
|
+
/* Frees all fields of ZopfliHash. */
|
53
|
+
void ZopfliCleanHash(ZopfliHash* h);
|
54
|
+
|
55
|
+
/*
|
56
|
+
Updates the hash values based on the current position in the array. All calls
|
57
|
+
to this must be made for consecutive bytes.
|
58
|
+
*/
|
59
|
+
void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
|
60
|
+
ZopfliHash* h);
|
61
|
+
|
62
|
+
/*
|
63
|
+
Prepopulates hash:
|
64
|
+
Fills in the initial values in the hash, before ZopfliUpdateHash can be used
|
65
|
+
correctly.
|
66
|
+
*/
|
67
|
+
void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
|
68
|
+
ZopfliHash* h);
|
69
|
+
|
70
|
+
#endif /* ZOPFLI_HASH_H_ */
|
@@ -0,0 +1,251 @@
|
|
1
|
+
/*
|
2
|
+
Copyright 2011 Google Inc. All Rights Reserved.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
15
|
+
|
16
|
+
Author: lode.vandevenne@gmail.com (Lode Vandevenne)
|
17
|
+
Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
|
18
|
+
*/
|
19
|
+
|
20
|
+
/*
|
21
|
+
Bounded package merge algorithm, based on the paper
|
22
|
+
"A Fast and Space-Economical Algorithm for Length-Limited Coding
|
23
|
+
Jyrki Katajainen, Alistair Moffat, Andrew Turpin".
|
24
|
+
*/
|
25
|
+
|
26
|
+
#include "katajainen.h"
|
27
|
+
#include <assert.h>
|
28
|
+
#include <stdlib.h>
|
29
|
+
|
30
|
+
typedef struct Node Node;
|
31
|
+
|
32
|
+
/*
|
33
|
+
Nodes forming chains. Also used to represent leaves.
|
34
|
+
*/
|
35
|
+
struct Node {
|
36
|
+
size_t weight; /* Total weight (symbol count) of this chain. */
|
37
|
+
Node* tail; /* Previous node(s) of this chain, or 0 if none. */
|
38
|
+
int count; /* Leaf symbol index, or number of leaves before this chain. */
|
39
|
+
char inuse; /* Tracking for garbage collection. */
|
40
|
+
};
|
41
|
+
|
42
|
+
/*
|
43
|
+
Memory pool for nodes.
|
44
|
+
*/
|
45
|
+
typedef struct NodePool {
|
46
|
+
Node* nodes; /* The pool. */
|
47
|
+
Node* next; /* Pointer to a possibly free node in the pool. */
|
48
|
+
int size; /* Size of the memory pool. */
|
49
|
+
} NodePool;
|
50
|
+
|
51
|
+
/*
|
52
|
+
Initializes a chain node with the given values and marks it as in use.
|
53
|
+
*/
|
54
|
+
static void InitNode(size_t weight, int count, Node* tail, Node* node) {
|
55
|
+
node->weight = weight;
|
56
|
+
node->count = count;
|
57
|
+
node->tail = tail;
|
58
|
+
node->inuse = 1;
|
59
|
+
}
|
60
|
+
|
61
|
+
/*
|
62
|
+
Finds a free location in the memory pool. Performs garbage collection if needed.
|
63
|
+
lists: If given, used to mark in-use nodes during garbage collection.
|
64
|
+
maxbits: Size of lists.
|
65
|
+
pool: Memory pool to get free node from.
|
66
|
+
*/
|
67
|
+
static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) {
|
68
|
+
for (;;) {
|
69
|
+
if (pool->next >= &pool->nodes[pool->size]) {
|
70
|
+
/* Garbage collection. */
|
71
|
+
int i;
|
72
|
+
for (i = 0; i < pool->size; i++) {
|
73
|
+
pool->nodes[i].inuse = 0;
|
74
|
+
}
|
75
|
+
if (lists) {
|
76
|
+
for (i = 0; i < maxbits * 2; i++) {
|
77
|
+
Node* node;
|
78
|
+
for (node = lists[i / 2][i % 2]; node; node = node->tail) {
|
79
|
+
node->inuse = 1;
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
pool->next = &pool->nodes[0];
|
84
|
+
}
|
85
|
+
if (!pool->next->inuse) break; /* Found one. */
|
86
|
+
pool->next++;
|
87
|
+
}
|
88
|
+
return pool->next++;
|
89
|
+
}
|
90
|
+
|
91
|
+
|
92
|
+
/*
|
93
|
+
Performs a Boundary Package-Merge step. Puts a new chain in the given list. The
|
94
|
+
new chain is, depending on the weights, a leaf or a combination of two chains
|
95
|
+
from the previous list.
|
96
|
+
lists: The lists of chains.
|
97
|
+
maxbits: Number of lists.
|
98
|
+
leaves: The leaves, one per symbol.
|
99
|
+
numsymbols: Number of leaves.
|
100
|
+
pool: the node memory pool.
|
101
|
+
index: The index of the list in which a new chain or leaf is required.
|
102
|
+
final: Whether this is the last time this function is called. If it is then it
|
103
|
+
is no more needed to recursively call self.
|
104
|
+
*/
|
105
|
+
static void BoundaryPM(Node* (*lists)[2], int maxbits,
|
106
|
+
Node* leaves, int numsymbols, NodePool* pool, int index, char final) {
|
107
|
+
Node* newchain;
|
108
|
+
Node* oldchain;
|
109
|
+
int lastcount = lists[index][1]->count; /* Count of last chain of list. */
|
110
|
+
|
111
|
+
if (index == 0 && lastcount >= numsymbols) return;
|
112
|
+
|
113
|
+
newchain = GetFreeNode(lists, maxbits, pool);
|
114
|
+
oldchain = lists[index][1];
|
115
|
+
|
116
|
+
/* These are set up before the recursive calls below, so that there is a list
|
117
|
+
pointing to the new node, to let the garbage collection know it's in use. */
|
118
|
+
lists[index][0] = oldchain;
|
119
|
+
lists[index][1] = newchain;
|
120
|
+
|
121
|
+
if (index == 0) {
|
122
|
+
/* New leaf node in list 0. */
|
123
|
+
InitNode(leaves[lastcount].weight, lastcount + 1, 0, newchain);
|
124
|
+
} else {
|
125
|
+
size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
|
126
|
+
if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
|
127
|
+
/* New leaf inserted in list, so count is incremented. */
|
128
|
+
InitNode(leaves[lastcount].weight, lastcount + 1, oldchain->tail,
|
129
|
+
newchain);
|
130
|
+
} else {
|
131
|
+
InitNode(sum, lastcount, lists[index - 1][1], newchain);
|
132
|
+
if (!final) {
|
133
|
+
/* Two lookahead chains of previous list used up, create new ones. */
|
134
|
+
BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
|
135
|
+
BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
|
136
|
+
}
|
137
|
+
}
|
138
|
+
}
|
139
|
+
}
|
140
|
+
|
141
|
+
/*
|
142
|
+
Initializes each list with as lookahead chains the two leaves with lowest
|
143
|
+
weights.
|
144
|
+
*/
|
145
|
+
static void InitLists(
|
146
|
+
NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) {
|
147
|
+
int i;
|
148
|
+
Node* node0 = GetFreeNode(0, maxbits, pool);
|
149
|
+
Node* node1 = GetFreeNode(0, maxbits, pool);
|
150
|
+
InitNode(leaves[0].weight, 1, 0, node0);
|
151
|
+
InitNode(leaves[1].weight, 2, 0, node1);
|
152
|
+
for (i = 0; i < maxbits; i++) {
|
153
|
+
lists[i][0] = node0;
|
154
|
+
lists[i][1] = node1;
|
155
|
+
}
|
156
|
+
}
|
157
|
+
|
158
|
+
/*
|
159
|
+
Converts result of boundary package-merge to the bitlengths. The result in the
|
160
|
+
last chain of the last list contains the amount of active leaves in each list.
|
161
|
+
chain: Chain to extract the bit length from (last chain from last list).
|
162
|
+
*/
|
163
|
+
static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) {
|
164
|
+
Node* node;
|
165
|
+
for (node = chain; node; node = node->tail) {
|
166
|
+
int i;
|
167
|
+
for (i = 0; i < node->count; i++) {
|
168
|
+
bitlengths[leaves[i].count]++;
|
169
|
+
}
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
/*
|
174
|
+
Comparator for sorting the leaves. Has the function signature for qsort.
|
175
|
+
*/
|
176
|
+
static int LeafComparator(const void* a, const void* b) {
|
177
|
+
return ((const Node*)a)->weight - ((const Node*)b)->weight;
|
178
|
+
}
|
179
|
+
|
180
|
+
int ZopfliLengthLimitedCodeLengths(
|
181
|
+
const size_t* frequencies, int n, int maxbits, unsigned* bitlengths) {
|
182
|
+
NodePool pool;
|
183
|
+
int i;
|
184
|
+
int numsymbols = 0; /* Amount of symbols with frequency > 0. */
|
185
|
+
int numBoundaryPMRuns;
|
186
|
+
|
187
|
+
/* Array of lists of chains. Each list requires only two lookahead chains at
|
188
|
+
a time, so each list is a array of two Node*'s. */
|
189
|
+
Node* (*lists)[2];
|
190
|
+
|
191
|
+
/* One leaf per symbol. Only numsymbols leaves will be used. */
|
192
|
+
Node* leaves = (Node*)malloc(n * sizeof(*leaves));
|
193
|
+
|
194
|
+
/* Initialize all bitlengths at 0. */
|
195
|
+
for (i = 0; i < n; i++) {
|
196
|
+
bitlengths[i] = 0;
|
197
|
+
}
|
198
|
+
|
199
|
+
/* Count used symbols and place them in the leaves. */
|
200
|
+
for (i = 0; i < n; i++) {
|
201
|
+
if (frequencies[i]) {
|
202
|
+
leaves[numsymbols].weight = frequencies[i];
|
203
|
+
leaves[numsymbols].count = i; /* Index of symbol this leaf represents. */
|
204
|
+
numsymbols++;
|
205
|
+
}
|
206
|
+
}
|
207
|
+
|
208
|
+
/* Check special cases and error conditions. */
|
209
|
+
if ((1 << maxbits) < numsymbols) {
|
210
|
+
free(leaves);
|
211
|
+
return 1; /* Error, too few maxbits to represent symbols. */
|
212
|
+
}
|
213
|
+
if (numsymbols == 0) {
|
214
|
+
free(leaves);
|
215
|
+
return 0; /* No symbols at all. OK. */
|
216
|
+
}
|
217
|
+
if (numsymbols == 1) {
|
218
|
+
bitlengths[leaves[0].count] = 1;
|
219
|
+
free(leaves);
|
220
|
+
return 0; /* Only one symbol, give it bitlength 1, not 0. OK. */
|
221
|
+
}
|
222
|
+
|
223
|
+
/* Sort the leaves from lightest to heaviest. */
|
224
|
+
qsort(leaves, numsymbols, sizeof(Node), LeafComparator);
|
225
|
+
|
226
|
+
/* Initialize node memory pool. */
|
227
|
+
pool.size = 2 * maxbits * (maxbits + 1);
|
228
|
+
pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes));
|
229
|
+
pool.next = pool.nodes;
|
230
|
+
for (i = 0; i < pool.size; i++) {
|
231
|
+
pool.nodes[i].inuse = 0;
|
232
|
+
}
|
233
|
+
|
234
|
+
lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists));
|
235
|
+
InitLists(&pool, leaves, maxbits, lists);
|
236
|
+
|
237
|
+
/* In the last list, 2 * numsymbols - 2 active chains need to be created. Two
|
238
|
+
are already created in the initialization. Each BoundaryPM run creates one. */
|
239
|
+
numBoundaryPMRuns = 2 * numsymbols - 4;
|
240
|
+
for (i = 0; i < numBoundaryPMRuns; i++) {
|
241
|
+
char final = i == numBoundaryPMRuns - 1;
|
242
|
+
BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final);
|
243
|
+
}
|
244
|
+
|
245
|
+
ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths);
|
246
|
+
|
247
|
+
free(lists);
|
248
|
+
free(leaves);
|
249
|
+
free(pool.nodes);
|
250
|
+
return 0; /* OK. */
|
251
|
+
}
|