zopfli 0.0.5 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +1 -1
  3. data/.travis.yml +24 -0
  4. data/Gemfile +2 -0
  5. data/Rakefile +8 -10
  6. data/ext/extconf.rb +1 -0
  7. data/ext/zopfli.c +37 -17
  8. data/lib/zopfli/version.rb +1 -1
  9. data/smoke.sh +9 -0
  10. data/{test → spec}/fixtures/alice29.txt +0 -0
  11. data/spec/spec_helper.rb +2 -0
  12. data/spec/zopfli_spec.rb +68 -0
  13. data/vendor/zopfli/src/zopfli/blocksplitter.c +34 -44
  14. data/vendor/zopfli/src/zopfli/blocksplitter.h +2 -6
  15. data/vendor/zopfli/src/zopfli/cache.c +3 -1
  16. data/vendor/zopfli/src/zopfli/deflate.c +351 -287
  17. data/vendor/zopfli/src/zopfli/deflate.h +8 -2
  18. data/vendor/zopfli/src/zopfli/gzip_container.c +54 -47
  19. data/vendor/zopfli/src/zopfli/hash.c +18 -10
  20. data/vendor/zopfli/src/zopfli/hash.h +6 -3
  21. data/vendor/zopfli/src/zopfli/katajainen.c +73 -62
  22. data/vendor/zopfli/src/zopfli/lz77.c +190 -42
  23. data/vendor/zopfli/src/zopfli/lz77.h +32 -19
  24. data/vendor/zopfli/src/zopfli/squeeze.c +75 -61
  25. data/vendor/zopfli/src/zopfli/squeeze.h +1 -0
  26. data/vendor/zopfli/src/zopfli/symbols.h +239 -0
  27. data/vendor/zopfli/src/zopfli/util.c +0 -178
  28. data/vendor/zopfli/src/zopfli/util.h +6 -23
  29. data/vendor/zopfli/src/zopfli/zlib_container.c +1 -1
  30. data/vendor/zopfli/src/zopfli/zopfli.h +1 -4
  31. data/vendor/zopfli/src/zopfli/zopfli_bin.c +11 -8
  32. data/zopfli.gemspec +8 -27
  33. metadata +13 -20
  34. data/test/test_zopfli_deflate.rb +0 -45
  35. data/vendor/zopfli/CONTRIBUTING.md +0 -24
  36. data/vendor/zopfli/CONTRIBUTORS +0 -8
  37. data/vendor/zopfli/Makefile +0 -42
  38. data/vendor/zopfli/README +0 -32
  39. data/vendor/zopfli/README.zopflipng +0 -35
  40. data/vendor/zopfli/src/zopflipng/lodepng/lodepng.cpp +0 -6252
  41. data/vendor/zopfli/src/zopflipng/lodepng/lodepng.h +0 -1716
  42. data/vendor/zopfli/src/zopflipng/lodepng/lodepng_util.cpp +0 -656
  43. data/vendor/zopfli/src/zopflipng/lodepng/lodepng_util.h +0 -151
  44. data/vendor/zopfli/src/zopflipng/zopflipng_bin.cc +0 -408
  45. data/vendor/zopfli/src/zopflipng/zopflipng_lib.cc +0 -492
  46. data/vendor/zopfli/src/zopflipng/zopflipng_lib.h +0 -134
@@ -25,6 +25,7 @@ Functions to compress according to the DEFLATE specification, using the
25
25
  "squeeze" LZ77 compression backend.
26
26
  */
27
27
 
28
+ #include "lz77.h"
28
29
  #include "zopfli.h"
29
30
 
30
31
  #ifdef __cplusplus
@@ -75,10 +76,15 @@ dists: ll77 distances
75
76
  lstart: start of block
76
77
  lend: end of block (not inclusive)
77
78
  */
78
- double ZopfliCalculateBlockSize(const unsigned short* litlens,
79
- const unsigned short* dists,
79
+ double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
80
80
  size_t lstart, size_t lend, int btype);
81
81
 
82
+ /*
83
+ Calculates block size in bits, automatically using the best btype.
84
+ */
85
+ double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
86
+ size_t lstart, size_t lend);
87
+
82
88
  #ifdef __cplusplus
83
89
  } // extern "C"
84
90
  #endif
@@ -24,56 +24,63 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
24
24
 
25
25
  #include "deflate.h"
26
26
 
27
- /* Table of CRCs of all 8-bit messages. */
28
- static unsigned long crc_table[256];
29
-
30
- /* Flag: has the table been computed? Initially false. */
31
- static int crc_table_computed = 0;
32
-
33
- /* Makes the table for a fast CRC. */
34
- static void MakeCRCTable() {
35
- unsigned long c;
36
- int n, k;
37
- for (n = 0; n < 256; n++) {
38
- c = (unsigned long) n;
39
- for (k = 0; k < 8; k++) {
40
- if (c & 1) {
41
- c = 0xedb88320L ^ (c >> 1);
42
- } else {
43
- c = c >> 1;
44
- }
45
- }
46
- crc_table[n] = c;
27
+ /* CRC polynomial: 0xedb88320 */
28
+ static const unsigned long crc32_table[256] = {
29
+ 0u, 1996959894u, 3993919788u, 2567524794u, 124634137u, 1886057615u,
30
+ 3915621685u, 2657392035u, 249268274u, 2044508324u, 3772115230u, 2547177864u,
31
+ 162941995u, 2125561021u, 3887607047u, 2428444049u, 498536548u, 1789927666u,
32
+ 4089016648u, 2227061214u, 450548861u, 1843258603u, 4107580753u, 2211677639u,
33
+ 325883990u, 1684777152u, 4251122042u, 2321926636u, 335633487u, 1661365465u,
34
+ 4195302755u, 2366115317u, 997073096u, 1281953886u, 3579855332u, 2724688242u,
35
+ 1006888145u, 1258607687u, 3524101629u, 2768942443u, 901097722u, 1119000684u,
36
+ 3686517206u, 2898065728u, 853044451u, 1172266101u, 3705015759u, 2882616665u,
37
+ 651767980u, 1373503546u, 3369554304u, 3218104598u, 565507253u, 1454621731u,
38
+ 3485111705u, 3099436303u, 671266974u, 1594198024u, 3322730930u, 2970347812u,
39
+ 795835527u, 1483230225u, 3244367275u, 3060149565u, 1994146192u, 31158534u,
40
+ 2563907772u, 4023717930u, 1907459465u, 112637215u, 2680153253u, 3904427059u,
41
+ 2013776290u, 251722036u, 2517215374u, 3775830040u, 2137656763u, 141376813u,
42
+ 2439277719u, 3865271297u, 1802195444u, 476864866u, 2238001368u, 4066508878u,
43
+ 1812370925u, 453092731u, 2181625025u, 4111451223u, 1706088902u, 314042704u,
44
+ 2344532202u, 4240017532u, 1658658271u, 366619977u, 2362670323u, 4224994405u,
45
+ 1303535960u, 984961486u, 2747007092u, 3569037538u, 1256170817u, 1037604311u,
46
+ 2765210733u, 3554079995u, 1131014506u, 879679996u, 2909243462u, 3663771856u,
47
+ 1141124467u, 855842277u, 2852801631u, 3708648649u, 1342533948u, 654459306u,
48
+ 3188396048u, 3373015174u, 1466479909u, 544179635u, 3110523913u, 3462522015u,
49
+ 1591671054u, 702138776u, 2966460450u, 3352799412u, 1504918807u, 783551873u,
50
+ 3082640443u, 3233442989u, 3988292384u, 2596254646u, 62317068u, 1957810842u,
51
+ 3939845945u, 2647816111u, 81470997u, 1943803523u, 3814918930u, 2489596804u,
52
+ 225274430u, 2053790376u, 3826175755u, 2466906013u, 167816743u, 2097651377u,
53
+ 4027552580u, 2265490386u, 503444072u, 1762050814u, 4150417245u, 2154129355u,
54
+ 426522225u, 1852507879u, 4275313526u, 2312317920u, 282753626u, 1742555852u,
55
+ 4189708143u, 2394877945u, 397917763u, 1622183637u, 3604390888u, 2714866558u,
56
+ 953729732u, 1340076626u, 3518719985u, 2797360999u, 1068828381u, 1219638859u,
57
+ 3624741850u, 2936675148u, 906185462u, 1090812512u, 3747672003u, 2825379669u,
58
+ 829329135u, 1181335161u, 3412177804u, 3160834842u, 628085408u, 1382605366u,
59
+ 3423369109u, 3138078467u, 570562233u, 1426400815u, 3317316542u, 2998733608u,
60
+ 733239954u, 1555261956u, 3268935591u, 3050360625u, 752459403u, 1541320221u,
61
+ 2607071920u, 3965973030u, 1969922972u, 40735498u, 2617837225u, 3943577151u,
62
+ 1913087877u, 83908371u, 2512341634u, 3803740692u, 2075208622u, 213261112u,
63
+ 2463272603u, 3855990285u, 2094854071u, 198958881u, 2262029012u, 4057260610u,
64
+ 1759359992u, 534414190u, 2176718541u, 4139329115u, 1873836001u, 414664567u,
65
+ 2282248934u, 4279200368u, 1711684554u, 285281116u, 2405801727u, 4167216745u,
66
+ 1634467795u, 376229701u, 2685067896u, 3608007406u, 1308918612u, 956543938u,
67
+ 2808555105u, 3495958263u, 1231636301u, 1047427035u, 2932959818u, 3654703836u,
68
+ 1088359270u, 936918000u, 2847714899u, 3736837829u, 1202900863u, 817233897u,
69
+ 3183342108u, 3401237130u, 1404277552u, 615818150u, 3134207493u, 3453421203u,
70
+ 1423857449u, 601450431u, 3009837614u, 3294710456u, 1567103746u, 711928724u,
71
+ 3020668471u, 3272380065u, 1510334235u, 755167117u
72
+ };
73
+
74
+ /* Returns the CRC32 */
75
+ static unsigned long CRC(const unsigned char* data, size_t size) {
76
+ unsigned long result = 0xffffffffu;
77
+ for (; size > 0; size--) {
78
+ result = crc32_table[(result ^ *(data++)) & 0xff] ^ (result >> 8);
47
79
  }
48
- crc_table_computed = 1;
80
+ return result ^ 0xffffffffu;
49
81
  }
50
82
 
51
-
52
- /*
53
- Updates a running crc with the bytes buf[0..len-1] and returns
54
- the updated crc. The crc should be initialized to zero.
55
- */
56
- static unsigned long UpdateCRC(unsigned long crc,
57
- const unsigned char *buf, size_t len) {
58
- unsigned long c = crc ^ 0xffffffffL;
59
- unsigned n;
60
-
61
- if (!crc_table_computed)
62
- MakeCRCTable();
63
- for (n = 0; n < len; n++) {
64
- c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
65
- }
66
- return c ^ 0xffffffffL;
67
- }
68
-
69
- /* Returns the CRC of the bytes buf[0..len-1]. */
70
- static unsigned long CRC(const unsigned char* buf, int len) {
71
- return UpdateCRC(0L, buf, len);
72
- }
73
-
74
- /*
75
- Compresses the data according to the gzip specification.
76
- */
83
+ /* Compresses the data according to the gzip specification, RFC 1952. */
77
84
  void ZopfliGzipCompress(const ZopfliOptions* options,
78
85
  const unsigned char* in, size_t insize,
79
86
  unsigned char** out, size_t* outsize) {
@@ -26,13 +26,26 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
26
26
  #define HASH_SHIFT 5
27
27
  #define HASH_MASK 32767
28
28
 
29
- void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
30
- size_t i;
31
-
32
- h->val = 0;
29
+ void ZopfliAllocHash(size_t window_size, ZopfliHash* h) {
33
30
  h->head = (int*)malloc(sizeof(*h->head) * 65536);
34
31
  h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size);
35
32
  h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size);
33
+
34
+ #ifdef ZOPFLI_HASH_SAME
35
+ h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
36
+ #endif
37
+
38
+ #ifdef ZOPFLI_HASH_SAME_HASH
39
+ h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
40
+ h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
41
+ h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
42
+ #endif
43
+ }
44
+
45
+ void ZopfliResetHash(size_t window_size, ZopfliHash* h) {
46
+ size_t i;
47
+
48
+ h->val = 0;
36
49
  for (i = 0; i < 65536; i++) {
37
50
  h->head[i] = -1; /* -1 indicates no head so far. */
38
51
  }
@@ -42,7 +55,6 @@ void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
42
55
  }
43
56
 
44
57
  #ifdef ZOPFLI_HASH_SAME
45
- h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
46
58
  for (i = 0; i < window_size; i++) {
47
59
  h->same[i] = 0;
48
60
  }
@@ -50,9 +62,6 @@ void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
50
62
 
51
63
  #ifdef ZOPFLI_HASH_SAME_HASH
52
64
  h->val2 = 0;
53
- h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
54
- h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
55
- h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
56
65
  for (i = 0; i < 65536; i++) {
57
66
  h->head2[i] = -1;
58
67
  }
@@ -129,7 +138,6 @@ void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
129
138
 
130
139
  void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
131
140
  ZopfliHash* h) {
132
- (void)end;
133
141
  UpdateHashValue(h, array[pos + 0]);
134
- UpdateHashValue(h, array[pos + 1]);
142
+ if (pos + 1 < end) UpdateHashValue(h, array[pos + 1]);
135
143
  }
@@ -46,10 +46,13 @@ typedef struct ZopfliHash {
46
46
  #endif
47
47
  } ZopfliHash;
48
48
 
49
- /* Allocates and initializes all fields of ZopfliHash. */
50
- void ZopfliInitHash(size_t window_size, ZopfliHash* h);
49
+ /* Allocates ZopfliHash memory. */
50
+ void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
51
51
 
52
- /* Frees all fields of ZopfliHash. */
52
+ /* Resets all fields of ZopfliHash. */
53
+ void ZopfliResetHash(size_t window_size, ZopfliHash* h);
54
+
55
+ /* Frees ZopfliHash memory. */
53
56
  void ZopfliCleanHash(ZopfliHash* h);
54
57
 
55
58
  /*
@@ -26,6 +26,7 @@ Jyrki Katajainen, Alistair Moffat, Andrew Turpin".
26
26
  #include "katajainen.h"
27
27
  #include <assert.h>
28
28
  #include <stdlib.h>
29
+ #include <limits.h>
29
30
 
30
31
  typedef struct Node Node;
31
32
 
@@ -36,16 +37,13 @@ struct Node {
36
37
  size_t weight; /* Total weight (symbol count) of this chain. */
37
38
  Node* tail; /* Previous node(s) of this chain, or 0 if none. */
38
39
  int count; /* Leaf symbol index, or number of leaves before this chain. */
39
- char inuse; /* Tracking for garbage collection. */
40
40
  };
41
41
 
42
42
  /*
43
43
  Memory pool for nodes.
44
44
  */
45
45
  typedef struct NodePool {
46
- Node* nodes; /* The pool. */
47
- Node* next; /* Pointer to a possibly free node in the pool. */
48
- int size; /* Size of the memory pool. */
46
+ Node* next; /* Pointer to a free node in the pool. */
49
47
  } NodePool;
50
48
 
51
49
  /*
@@ -55,40 +53,8 @@ static void InitNode(size_t weight, int count, Node* tail, Node* node) {
55
53
  node->weight = weight;
56
54
  node->count = count;
57
55
  node->tail = tail;
58
- node->inuse = 1;
59
56
  }
60
57
 
61
- /*
62
- Finds a free location in the memory pool. Performs garbage collection if needed.
63
- lists: If given, used to mark in-use nodes during garbage collection.
64
- maxbits: Size of lists.
65
- pool: Memory pool to get free node from.
66
- */
67
- static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) {
68
- for (;;) {
69
- if (pool->next >= &pool->nodes[pool->size]) {
70
- /* Garbage collection. */
71
- int i;
72
- for (i = 0; i < pool->size; i++) {
73
- pool->nodes[i].inuse = 0;
74
- }
75
- if (lists) {
76
- for (i = 0; i < maxbits * 2; i++) {
77
- Node* node;
78
- for (node = lists[i / 2][i % 2]; node; node = node->tail) {
79
- node->inuse = 1;
80
- }
81
- }
82
- }
83
- pool->next = &pool->nodes[0];
84
- }
85
- if (!pool->next->inuse) break; /* Found one. */
86
- pool->next++;
87
- }
88
- return pool->next++;
89
- }
90
-
91
-
92
58
  /*
93
59
  Performs a Boundary Package-Merge step. Puts a new chain in the given list. The
94
60
  new chain is, depending on the weights, a leaf or a combination of two chains
@@ -99,18 +65,16 @@ leaves: The leaves, one per symbol.
99
65
  numsymbols: Number of leaves.
100
66
  pool: the node memory pool.
101
67
  index: The index of the list in which a new chain or leaf is required.
102
- final: Whether this is the last time this function is called. If it is then it
103
- is no more needed to recursively call self.
104
68
  */
105
- static void BoundaryPM(Node* (*lists)[2], int maxbits,
106
- Node* leaves, int numsymbols, NodePool* pool, int index, char final) {
69
+ static void BoundaryPM(Node* (*lists)[2], Node* leaves, int numsymbols,
70
+ NodePool* pool, int index) {
107
71
  Node* newchain;
108
72
  Node* oldchain;
109
73
  int lastcount = lists[index][1]->count; /* Count of last chain of list. */
110
74
 
111
75
  if (index == 0 && lastcount >= numsymbols) return;
112
76
 
113
- newchain = GetFreeNode(lists, maxbits, pool);
77
+ newchain = pool->next++;
114
78
  oldchain = lists[index][1];
115
79
 
116
80
  /* These are set up before the recursive calls below, so that there is a list
@@ -129,15 +93,31 @@ static void BoundaryPM(Node* (*lists)[2], int maxbits,
129
93
  newchain);
130
94
  } else {
131
95
  InitNode(sum, lastcount, lists[index - 1][1], newchain);
132
- if (!final) {
133
- /* Two lookahead chains of previous list used up, create new ones. */
134
- BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
135
- BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
136
- }
96
+ /* Two lookahead chains of previous list used up, create new ones. */
97
+ BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
98
+ BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
137
99
  }
138
100
  }
139
101
  }
140
102
 
103
+ static void BoundaryPMFinal(Node* (*lists)[2],
104
+ Node* leaves, int numsymbols, NodePool* pool, int index) {
105
+ int lastcount = lists[index][1]->count; /* Count of last chain of list. */
106
+
107
+ size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
108
+
109
+ if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
110
+ Node* newchain = pool->next;
111
+ Node* oldchain = lists[index][1]->tail;
112
+
113
+ lists[index][1] = newchain;
114
+ newchain->count = lastcount + 1;
115
+ newchain->tail = oldchain;
116
+ } else {
117
+ lists[index][1]->tail = lists[index - 1][1];
118
+ }
119
+ }
120
+
141
121
  /*
142
122
  Initializes each list with as lookahead chains the two leaves with lowest
143
123
  weights.
@@ -145,8 +125,8 @@ weights.
145
125
  static void InitLists(
146
126
  NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) {
147
127
  int i;
148
- Node* node0 = GetFreeNode(0, maxbits, pool);
149
- Node* node1 = GetFreeNode(0, maxbits, pool);
128
+ Node* node0 = pool->next++;
129
+ Node* node1 = pool->next++;
150
130
  InitNode(leaves[0].weight, 1, 0, node0);
151
131
  InitNode(leaves[1].weight, 2, 0, node1);
152
132
  for (i = 0; i < maxbits; i++) {
@@ -161,12 +141,24 @@ last chain of the last list contains the amount of active leaves in each list.
161
141
  chain: Chain to extract the bit length from (last chain from last list).
162
142
  */
163
143
  static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) {
144
+ int counts[16] = {0};
145
+ unsigned end = 16;
146
+ unsigned ptr = 15;
147
+ unsigned value = 1;
164
148
  Node* node;
149
+ int val;
150
+
165
151
  for (node = chain; node; node = node->tail) {
166
- int i;
167
- for (i = 0; i < node->count; i++) {
168
- bitlengths[leaves[i].count]++;
152
+ counts[--end] = node->count;
153
+ }
154
+
155
+ val = counts[15];
156
+ while (ptr >= end) {
157
+ for (; val > counts[ptr - 1]; val--) {
158
+ bitlengths[leaves[val - 1].count] = value;
169
159
  }
160
+ ptr--;
161
+ value++;
170
162
  }
171
163
  }
172
164
 
@@ -183,6 +175,7 @@ int ZopfliLengthLimitedCodeLengths(
183
175
  int i;
184
176
  int numsymbols = 0; /* Amount of symbols with frequency > 0. */
185
177
  int numBoundaryPMRuns;
178
+ Node* nodes;
186
179
 
187
180
  /* Array of lists of chains. Each list requires only two lookahead chains at
188
181
  a time, so each list is a array of two Node*'s. */
@@ -219,33 +212,51 @@ int ZopfliLengthLimitedCodeLengths(
219
212
  free(leaves);
220
213
  return 0; /* Only one symbol, give it bitlength 1, not 0. OK. */
221
214
  }
215
+ if (numsymbols == 2) {
216
+ bitlengths[leaves[0].count]++;
217
+ bitlengths[leaves[1].count]++;
218
+ free(leaves);
219
+ return 0;
220
+ }
222
221
 
223
- /* Sort the leaves from lightest to heaviest. */
222
+ /* Sort the leaves from lightest to heaviest. Add count into the same
223
+ variable for stable sorting. */
224
+ for (i = 0; i < numsymbols; i++) {
225
+ if (leaves[i].weight >=
226
+ ((size_t)1 << (sizeof(leaves[0].weight) * CHAR_BIT - 9))) {
227
+ free(leaves);
228
+ return 1; /* Error, we need 9 bits for the count. */
229
+ }
230
+ leaves[i].weight = (leaves[i].weight << 9) | leaves[i].count;
231
+ }
224
232
  qsort(leaves, numsymbols, sizeof(Node), LeafComparator);
233
+ for (i = 0; i < numsymbols; i++) {
234
+ leaves[i].weight >>= 9;
235
+ }
225
236
 
226
- /* Initialize node memory pool. */
227
- pool.size = 2 * maxbits * (maxbits + 1);
228
- pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes));
229
- pool.next = pool.nodes;
230
- for (i = 0; i < pool.size; i++) {
231
- pool.nodes[i].inuse = 0;
237
+ if (numsymbols - 1 < maxbits) {
238
+ maxbits = numsymbols - 1;
232
239
  }
233
240
 
241
+ /* Initialize node memory pool. */
242
+ nodes = (Node*)malloc(maxbits * 2 * numsymbols * sizeof(Node));
243
+ pool.next = nodes;
244
+
234
245
  lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists));
235
246
  InitLists(&pool, leaves, maxbits, lists);
236
247
 
237
248
  /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two
238
249
  are already created in the initialization. Each BoundaryPM run creates one. */
239
250
  numBoundaryPMRuns = 2 * numsymbols - 4;
240
- for (i = 0; i < numBoundaryPMRuns; i++) {
241
- char final = i == numBoundaryPMRuns - 1;
242
- BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final);
251
+ for (i = 0; i < numBoundaryPMRuns - 1; i++) {
252
+ BoundaryPM(lists, leaves, numsymbols, &pool, maxbits - 1);
243
253
  }
254
+ BoundaryPMFinal(lists, leaves, numsymbols, &pool, maxbits - 1);
244
255
 
245
256
  ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths);
246
257
 
247
258
  free(lists);
248
259
  free(leaves);
249
- free(pool.nodes);
260
+ free(nodes);
250
261
  return 0; /* OK. */
251
262
  }
@@ -18,37 +18,76 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
18
18
  */
19
19
 
20
20
  #include "lz77.h"
21
+ #include "symbols.h"
21
22
  #include "util.h"
22
23
 
23
24
  #include <assert.h>
24
25
  #include <stdio.h>
25
26
  #include <stdlib.h>
26
27
 
27
- void ZopfliInitLZ77Store(ZopfliLZ77Store* store) {
28
+ void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store) {
28
29
  store->size = 0;
29
30
  store->litlens = 0;
30
31
  store->dists = 0;
32
+ store->pos = 0;
33
+ store->data = data;
34
+ store->ll_symbol = 0;
35
+ store->d_symbol = 0;
36
+ store->ll_counts = 0;
37
+ store->d_counts = 0;
31
38
  }
32
39
 
33
40
  void ZopfliCleanLZ77Store(ZopfliLZ77Store* store) {
34
41
  free(store->litlens);
35
42
  free(store->dists);
43
+ free(store->pos);
44
+ free(store->ll_symbol);
45
+ free(store->d_symbol);
46
+ free(store->ll_counts);
47
+ free(store->d_counts);
48
+ }
49
+
50
+ static size_t CeilDiv(size_t a, size_t b) {
51
+ return (a + b - 1) / b;
36
52
  }
37
53
 
38
54
  void ZopfliCopyLZ77Store(
39
55
  const ZopfliLZ77Store* source, ZopfliLZ77Store* dest) {
40
56
  size_t i;
57
+ size_t llsize = ZOPFLI_NUM_LL * CeilDiv(source->size, ZOPFLI_NUM_LL);
58
+ size_t dsize = ZOPFLI_NUM_D * CeilDiv(source->size, ZOPFLI_NUM_D);
41
59
  ZopfliCleanLZ77Store(dest);
60
+ ZopfliInitLZ77Store(source->data, dest);
42
61
  dest->litlens =
43
62
  (unsigned short*)malloc(sizeof(*dest->litlens) * source->size);
44
63
  dest->dists = (unsigned short*)malloc(sizeof(*dest->dists) * source->size);
45
-
46
- if (!dest->litlens || !dest->dists) exit(-1); /* Allocation failed. */
64
+ dest->pos = (size_t*)malloc(sizeof(*dest->pos) * source->size);
65
+ dest->ll_symbol =
66
+ (unsigned short*)malloc(sizeof(*dest->ll_symbol) * source->size);
67
+ dest->d_symbol =
68
+ (unsigned short*)malloc(sizeof(*dest->d_symbol) * source->size);
69
+ dest->ll_counts = (size_t*)malloc(sizeof(*dest->ll_counts) * llsize);
70
+ dest->d_counts = (size_t*)malloc(sizeof(*dest->d_counts) * dsize);
71
+
72
+ /* Allocation failed. */
73
+ if (!dest->litlens || !dest->dists) exit(-1);
74
+ if (!dest->pos) exit(-1);
75
+ if (!dest->ll_symbol || !dest->d_symbol) exit(-1);
76
+ if (!dest->ll_counts || !dest->d_counts) exit(-1);
47
77
 
48
78
  dest->size = source->size;
49
79
  for (i = 0; i < source->size; i++) {
50
80
  dest->litlens[i] = source->litlens[i];
51
81
  dest->dists[i] = source->dists[i];
82
+ dest->pos[i] = source->pos[i];
83
+ dest->ll_symbol[i] = source->ll_symbol[i];
84
+ dest->d_symbol[i] = source->d_symbol[i];
85
+ }
86
+ for (i = 0; i < llsize; i++) {
87
+ dest->ll_counts[i] = source->ll_counts[i];
88
+ }
89
+ for (i = 0; i < dsize; i++) {
90
+ dest->d_counts[i] = source->d_counts[i];
52
91
  }
53
92
  }
54
93
 
@@ -57,10 +96,149 @@ Appends the length and distance to the LZ77 arrays of the ZopfliLZ77Store.
57
96
  context must be a ZopfliLZ77Store*.
58
97
  */
59
98
  void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
60
- ZopfliLZ77Store* store) {
61
- size_t size2 = store->size; /* Needed for using ZOPFLI_APPEND_DATA twice. */
99
+ size_t pos, ZopfliLZ77Store* store) {
100
+ size_t i;
101
+ /* Needed for using ZOPFLI_APPEND_DATA multiple times. */
102
+ size_t origsize = store->size;
103
+ size_t llstart = ZOPFLI_NUM_LL * (origsize / ZOPFLI_NUM_LL);
104
+ size_t dstart = ZOPFLI_NUM_D * (origsize / ZOPFLI_NUM_D);
105
+
106
+ /* Everytime the index wraps around, a new cumulative histogram is made: we're
107
+ keeping one histogram value per LZ77 symbol rather than a full histogram for
108
+ each to save memory. */
109
+ if (origsize % ZOPFLI_NUM_LL == 0) {
110
+ size_t llsize = origsize;
111
+ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
112
+ ZOPFLI_APPEND_DATA(
113
+ origsize == 0 ? 0 : store->ll_counts[origsize - ZOPFLI_NUM_LL + i],
114
+ &store->ll_counts, &llsize);
115
+ }
116
+ }
117
+ if (origsize % ZOPFLI_NUM_D == 0) {
118
+ size_t dsize = origsize;
119
+ for (i = 0; i < ZOPFLI_NUM_D; i++) {
120
+ ZOPFLI_APPEND_DATA(
121
+ origsize == 0 ? 0 : store->d_counts[origsize - ZOPFLI_NUM_D + i],
122
+ &store->d_counts, &dsize);
123
+ }
124
+ }
125
+
62
126
  ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size);
63
- ZOPFLI_APPEND_DATA(dist, &store->dists, &size2);
127
+ store->size = origsize;
128
+ ZOPFLI_APPEND_DATA(dist, &store->dists, &store->size);
129
+ store->size = origsize;
130
+ ZOPFLI_APPEND_DATA(pos, &store->pos, &store->size);
131
+ assert(length < 259);
132
+
133
+ if (dist == 0) {
134
+ store->size = origsize;
135
+ ZOPFLI_APPEND_DATA(length, &store->ll_symbol, &store->size);
136
+ store->size = origsize;
137
+ ZOPFLI_APPEND_DATA(0, &store->d_symbol, &store->size);
138
+ store->ll_counts[llstart + length]++;
139
+ } else {
140
+ store->size = origsize;
141
+ ZOPFLI_APPEND_DATA(ZopfliGetLengthSymbol(length),
142
+ &store->ll_symbol, &store->size);
143
+ store->size = origsize;
144
+ ZOPFLI_APPEND_DATA(ZopfliGetDistSymbol(dist),
145
+ &store->d_symbol, &store->size);
146
+ store->ll_counts[llstart + ZopfliGetLengthSymbol(length)]++;
147
+ store->d_counts[dstart + ZopfliGetDistSymbol(dist)]++;
148
+ }
149
+ }
150
+
151
+ void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
152
+ ZopfliLZ77Store* target) {
153
+ size_t i;
154
+ for (i = 0; i < store->size; i++) {
155
+ ZopfliStoreLitLenDist(store->litlens[i], store->dists[i],
156
+ store->pos[i], target);
157
+ }
158
+ }
159
+
160
+ size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
161
+ size_t lstart, size_t lend) {
162
+ size_t l = lend - 1;
163
+ if (lstart == lend) return 0;
164
+ return lz77->pos[l] + ((lz77->dists[l] == 0) ?
165
+ 1 : lz77->litlens[l]) - lz77->pos[lstart];
166
+ }
167
+
168
+ static void ZopfliLZ77GetHistogramAt(const ZopfliLZ77Store* lz77, size_t lpos,
169
+ size_t* ll_counts, size_t* d_counts) {
170
+ /* The real histogram is created by using the histogram for this chunk, but
171
+ all superfluous values of this chunk subtracted. */
172
+ size_t llpos = ZOPFLI_NUM_LL * (lpos / ZOPFLI_NUM_LL);
173
+ size_t dpos = ZOPFLI_NUM_D * (lpos / ZOPFLI_NUM_D);
174
+ size_t i;
175
+ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
176
+ ll_counts[i] = lz77->ll_counts[llpos + i];
177
+ }
178
+ for (i = lpos + 1; i < llpos + ZOPFLI_NUM_LL && i < lz77->size; i++) {
179
+ ll_counts[lz77->ll_symbol[i]]--;
180
+ }
181
+ for (i = 0; i < ZOPFLI_NUM_D; i++) {
182
+ d_counts[i] = lz77->d_counts[dpos + i];
183
+ }
184
+ for (i = lpos + 1; i < dpos + ZOPFLI_NUM_D && i < lz77->size; i++) {
185
+ if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]--;
186
+ }
187
+ }
188
+
189
+ void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
190
+ size_t lstart, size_t lend,
191
+ size_t* ll_counts, size_t* d_counts) {
192
+ size_t i;
193
+ if (lstart + ZOPFLI_NUM_LL * 3 > lend) {
194
+ memset(ll_counts, 0, sizeof(*ll_counts) * ZOPFLI_NUM_LL);
195
+ memset(d_counts, 0, sizeof(*d_counts) * ZOPFLI_NUM_D);
196
+ for (i = lstart; i < lend; i++) {
197
+ ll_counts[lz77->ll_symbol[i]]++;
198
+ if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]++;
199
+ }
200
+ } else {
201
+ /* Subtract the cumulative histograms at the end and the start to get the
202
+ histogram for this range. */
203
+ ZopfliLZ77GetHistogramAt(lz77, lend - 1, ll_counts, d_counts);
204
+ if (lstart > 0) {
205
+ size_t ll_counts2[ZOPFLI_NUM_LL];
206
+ size_t d_counts2[ZOPFLI_NUM_D];
207
+ ZopfliLZ77GetHistogramAt(lz77, lstart - 1, ll_counts2, d_counts2);
208
+
209
+ for (i = 0; i < ZOPFLI_NUM_LL; i++) {
210
+ ll_counts[i] -= ll_counts2[i];
211
+ }
212
+ for (i = 0; i < ZOPFLI_NUM_D; i++) {
213
+ d_counts[i] -= d_counts2[i];
214
+ }
215
+ }
216
+ }
217
+ }
218
+
219
+ void ZopfliInitBlockState(const ZopfliOptions* options,
220
+ size_t blockstart, size_t blockend, int add_lmc,
221
+ ZopfliBlockState* s) {
222
+ s->options = options;
223
+ s->blockstart = blockstart;
224
+ s->blockend = blockend;
225
+ #ifdef ZOPFLI_LONGEST_MATCH_CACHE
226
+ if (add_lmc) {
227
+ s->lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache));
228
+ ZopfliInitCache(blockend - blockstart, s->lmc);
229
+ } else {
230
+ s->lmc = 0;
231
+ }
232
+ #endif
233
+ }
234
+
235
+ void ZopfliCleanBlockState(ZopfliBlockState* s) {
236
+ #ifdef ZOPFLI_LONGEST_MATCH_CACHE
237
+ if (s->lmc) {
238
+ ZopfliCleanCache(s->lmc);
239
+ free(s->lmc);
240
+ }
241
+ #endif
64
242
  }
65
243
 
66
244
  /*
@@ -365,7 +543,7 @@ void ZopfliFindLongestMatch(ZopfliBlockState* s, const ZopfliHash* h,
365
543
 
366
544
  void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
367
545
  size_t instart, size_t inend,
368
- ZopfliLZ77Store* store) {
546
+ ZopfliLZ77Store* store, ZopfliHash* h) {
369
547
  size_t i = 0, j;
370
548
  unsigned short leng;
371
549
  unsigned short dist;
@@ -374,9 +552,6 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
374
552
  ? instart - ZOPFLI_WINDOW_SIZE : 0;
375
553
  unsigned short dummysublen[259];
376
554
 
377
- ZopfliHash hash;
378
- ZopfliHash* h = &hash;
379
-
380
555
  #ifdef ZOPFLI_LAZY_MATCHING
381
556
  /* Lazy matching. */
382
557
  unsigned prev_length = 0;
@@ -387,7 +562,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
387
562
 
388
563
  if (instart == inend) return;
389
564
 
390
- ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h);
565
+ ZopfliResetHash(ZOPFLI_WINDOW_SIZE, h);
391
566
  ZopfliWarmupHash(in, windowstart, inend, h);
392
567
  for (i = windowstart; i < instart; i++) {
393
568
  ZopfliUpdateHash(in, i, inend, h);
@@ -406,7 +581,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
406
581
  if (match_available) {
407
582
  match_available = 0;
408
583
  if (lengthscore > prevlengthscore + 1) {
409
- ZopfliStoreLitLenDist(in[i - 1], 0, store);
584
+ ZopfliStoreLitLenDist(in[i - 1], 0, i - 1, store);
410
585
  if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) {
411
586
  match_available = 1;
412
587
  prev_length = leng;
@@ -420,7 +595,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
420
595
  lengthscore = prevlengthscore;
421
596
  /* Add to output. */
422
597
  ZopfliVerifyLenDist(in, inend, i - 1, dist, leng);
423
- ZopfliStoreLitLenDist(leng, dist, store);
598
+ ZopfliStoreLitLenDist(leng, dist, i - 1, store);
424
599
  for (j = 2; j < leng; j++) {
425
600
  assert(i < inend);
426
601
  i++;
@@ -441,10 +616,10 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
441
616
  /* Add to output. */
442
617
  if (lengthscore >= ZOPFLI_MIN_MATCH) {
443
618
  ZopfliVerifyLenDist(in, inend, i, dist, leng);
444
- ZopfliStoreLitLenDist(leng, dist, store);
619
+ ZopfliStoreLitLenDist(leng, dist, i, store);
445
620
  } else {
446
621
  leng = 1;
447
- ZopfliStoreLitLenDist(in[i], 0, store);
622
+ ZopfliStoreLitLenDist(in[i], 0, i, store);
448
623
  }
449
624
  for (j = 1; j < leng; j++) {
450
625
  assert(i < inend);
@@ -452,31 +627,4 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
452
627
  ZopfliUpdateHash(in, i, inend, h);
453
628
  }
454
629
  }
455
-
456
- ZopfliCleanHash(h);
457
- }
458
-
459
- void ZopfliLZ77Counts(const unsigned short* litlens,
460
- const unsigned short* dists,
461
- size_t start, size_t end,
462
- size_t* ll_count, size_t* d_count) {
463
- size_t i;
464
-
465
- for (i = 0; i < 288; i++) {
466
- ll_count[i] = 0;
467
- }
468
- for (i = 0; i < 32; i++) {
469
- d_count[i] = 0;
470
- }
471
-
472
- for (i = start; i < end; i++) {
473
- if (dists[i] == 0) {
474
- ll_count[litlens[i]]++;
475
- } else {
476
- ll_count[ZopfliGetLengthSymbol(litlens[i])]++;
477
- d_count[ZopfliGetDistSymbol(dists[i])]++;
478
- }
479
- }
480
-
481
- ll_count[256] = 1; /* End symbol. */
482
630
  }