zopfli 0.0.2 → 0.0.8

data/vendor/zopfli/{deflate.h → src/zopfli/deflate.h}

@@ -25,8 +25,13 @@ Functions to compress according to the DEFLATE specification, using the
 "squeeze" LZ77 compression backend.
 */
 
+#include "lz77.h"
 #include "zopfli.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /*
 Compresses according to the deflate specification and append the compressed
 result to the output.
@@ -71,7 +76,17 @@ dists: ll77 distances
 lstart: start of block
 lend: end of block (not inclusive)
 */
-double ZopfliCalculateBlockSize(const unsigned short* litlens,
-                                const unsigned short* dists,
+double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
                                 size_t lstart, size_t lend, int btype);
+
+/*
+Calculates block size in bits, automatically using the best btype.
+*/
+double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
+                                        size_t lstart, size_t lend);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
 #endif  /* ZOPFLI_DEFLATE_H_ */

data/vendor/zopfli/src/zopfli/gzip_container.c

@@ -0,0 +1,124 @@
+/*
+Copyright 2013 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Author: lode.vandevenne@gmail.com (Lode Vandevenne)
+Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
+*/
+
+#include "gzip_container.h"
+#include "util.h"
+
+#include <stdio.h>
+
+#include "deflate.h"
+
+/* CRC polynomial: 0xedb88320 */
+static const unsigned long crc32_table[256] = {
+           0u, 1996959894u, 3993919788u, 2567524794u,  124634137u, 1886057615u,
+  3915621685u, 2657392035u,  249268274u, 2044508324u, 3772115230u, 2547177864u,
+   162941995u, 2125561021u, 3887607047u, 2428444049u,  498536548u, 1789927666u,
+  4089016648u, 2227061214u,  450548861u, 1843258603u, 4107580753u, 2211677639u,
+   325883990u, 1684777152u, 4251122042u, 2321926636u,  335633487u, 1661365465u,
+  4195302755u, 2366115317u,  997073096u, 1281953886u, 3579855332u, 2724688242u,
+  1006888145u, 1258607687u, 3524101629u, 2768942443u,  901097722u, 1119000684u,
+  3686517206u, 2898065728u,  853044451u, 1172266101u, 3705015759u, 2882616665u,
+   651767980u, 1373503546u, 3369554304u, 3218104598u,  565507253u, 1454621731u,
+  3485111705u, 3099436303u,  671266974u, 1594198024u, 3322730930u, 2970347812u,
+   795835527u, 1483230225u, 3244367275u, 3060149565u, 1994146192u,   31158534u,
+  2563907772u, 4023717930u, 1907459465u,  112637215u, 2680153253u, 3904427059u,
+  2013776290u,  251722036u, 2517215374u, 3775830040u, 2137656763u,  141376813u,
+  2439277719u, 3865271297u, 1802195444u,  476864866u, 2238001368u, 4066508878u,
+  1812370925u,  453092731u, 2181625025u, 4111451223u, 1706088902u,  314042704u,
+  2344532202u, 4240017532u, 1658658271u,  366619977u, 2362670323u, 4224994405u,
+  1303535960u,  984961486u, 2747007092u, 3569037538u, 1256170817u, 1037604311u,
+  2765210733u, 3554079995u, 1131014506u,  879679996u, 2909243462u, 3663771856u,
+  1141124467u,  855842277u, 2852801631u, 3708648649u, 1342533948u,  654459306u,
+  3188396048u, 3373015174u, 1466479909u,  544179635u, 3110523913u, 3462522015u,
+  1591671054u,  702138776u, 2966460450u, 3352799412u, 1504918807u,  783551873u,
+  3082640443u, 3233442989u, 3988292384u, 2596254646u,   62317068u, 1957810842u,
+  3939845945u, 2647816111u,   81470997u, 1943803523u, 3814918930u, 2489596804u,
+   225274430u, 2053790376u, 3826175755u, 2466906013u,  167816743u, 2097651377u,
+  4027552580u, 2265490386u,  503444072u, 1762050814u, 4150417245u, 2154129355u,
+   426522225u, 1852507879u, 4275313526u, 2312317920u,  282753626u, 1742555852u,
+  4189708143u, 2394877945u,  397917763u, 1622183637u, 3604390888u, 2714866558u,
+   953729732u, 1340076626u, 3518719985u, 2797360999u, 1068828381u, 1219638859u,
+  3624741850u, 2936675148u,  906185462u, 1090812512u, 3747672003u, 2825379669u,
+   829329135u, 1181335161u, 3412177804u, 3160834842u,  628085408u, 1382605366u,
+  3423369109u, 3138078467u,  570562233u, 1426400815u, 3317316542u, 2998733608u,
+   733239954u, 1555261956u, 3268935591u, 3050360625u,  752459403u, 1541320221u,
+  2607071920u, 3965973030u, 1969922972u,   40735498u, 2617837225u, 3943577151u,
+  1913087877u,   83908371u, 2512341634u, 3803740692u, 2075208622u,  213261112u,
+  2463272603u, 3855990285u, 2094854071u,  198958881u, 2262029012u, 4057260610u,
+  1759359992u,  534414190u, 2176718541u, 4139329115u, 1873836001u,  414664567u,
+  2282248934u, 4279200368u, 1711684554u,  285281116u, 2405801727u, 4167216745u,
+  1634467795u,  376229701u, 2685067896u, 3608007406u, 1308918612u,  956543938u,
+  2808555105u, 3495958263u, 1231636301u, 1047427035u, 2932959818u, 3654703836u,
+  1088359270u,  936918000u, 2847714899u, 3736837829u, 1202900863u,  817233897u,
+  3183342108u, 3401237130u, 1404277552u,  615818150u, 3134207493u, 3453421203u,
+  1423857449u,  601450431u, 3009837614u, 3294710456u, 1567103746u,  711928724u,
+  3020668471u, 3272380065u, 1510334235u,  755167117u
+};
+
+/* Returns the CRC32 */
+static unsigned long CRC(const unsigned char* data, size_t size) {
+  unsigned long result = 0xffffffffu;
+  for (; size > 0; size--) {
+    result = crc32_table[(result ^ *(data++)) & 0xff] ^ (result >> 8);
+  }
+  return result ^ 0xffffffffu;
+}
+
+/* Compresses the data according to the gzip specification, RFC 1952. */
+void ZopfliGzipCompress(const ZopfliOptions* options,
+                        const unsigned char* in, size_t insize,
+                        unsigned char** out, size_t* outsize) {
+  unsigned long crcvalue = CRC(in, insize);
+  unsigned char bp = 0;
+
+  ZOPFLI_APPEND_DATA(31, out, outsize);  /* ID1 */
+  ZOPFLI_APPEND_DATA(139, out, outsize);  /* ID2 */
+  ZOPFLI_APPEND_DATA(8, out, outsize);  /* CM */
+  ZOPFLI_APPEND_DATA(0, out, outsize);  /* FLG */
+  /* MTIME */
+  ZOPFLI_APPEND_DATA(0, out, outsize);
+  ZOPFLI_APPEND_DATA(0, out, outsize);
+  ZOPFLI_APPEND_DATA(0, out, outsize);
+  ZOPFLI_APPEND_DATA(0, out, outsize);
+
+  ZOPFLI_APPEND_DATA(2, out, outsize);  /* XFL, 2 indicates best compression. */
+  ZOPFLI_APPEND_DATA(3, out, outsize);  /* OS follows Unix conventions. */
+
+  ZopfliDeflate(options, 2 /* Dynamic block */, 1,
+                in, insize, &bp, out, outsize);
+
+  /* CRC */
+  ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize);
+  ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize);
+  ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize);
+  ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize);
+
+  /* ISIZE */
+  ZOPFLI_APPEND_DATA(insize % 256, out, outsize);
+  ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize);
+  ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize);
+  ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize);
+
+  if (options->verbose) {
+    fprintf(stderr,
+            "Original Size: %d, Gzip: %d, Compression: %f%% Removed\n",
+            (int)insize, (int)*outsize,
+            100.0 * (double)(insize - *outsize) / (double)insize);
+  }
+}

data/vendor/zopfli/{gzip_container.h → src/zopfli/gzip_container.h}

@@ -26,6 +26,10 @@ Functions to compress according to the Gzip specification.
 
 #include "zopfli.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /*
 Compresses according to the gzip specification and append the compressed
 result to the output.
@@ -39,4 +43,8 @@ void ZopfliGzipCompress(const ZopfliOptions* options,
                         const unsigned char* in, size_t insize,
                         unsigned char** out, size_t* outsize);
 
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
 #endif  /* ZOPFLI_GZIP_H_ */

data/vendor/zopfli/{hash.c → src/zopfli/hash.c}

@@ -26,13 +26,26 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
 #define HASH_SHIFT 5
 #define HASH_MASK 32767
 
-void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
-  size_t i;
-
-  h->val = 0;
+void ZopfliAllocHash(size_t window_size, ZopfliHash* h) {
   h->head = (int*)malloc(sizeof(*h->head) * 65536);
   h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size);
   h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size);
+
+#ifdef ZOPFLI_HASH_SAME
+  h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
+#endif
+
+#ifdef ZOPFLI_HASH_SAME_HASH
+  h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
+  h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
+  h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
+#endif
+}
+
+void ZopfliResetHash(size_t window_size, ZopfliHash* h) {
+  size_t i;
+
+  h->val = 0;
   for (i = 0; i < 65536; i++) {
     h->head[i] = -1;  /* -1 indicates no head so far. */
   }
@@ -42,7 +55,6 @@ void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
   }
 
 #ifdef ZOPFLI_HASH_SAME
-  h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
   for (i = 0; i < window_size; i++) {
     h->same[i] = 0;
   }
@@ -50,9 +62,6 @@ void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
 
 #ifdef ZOPFLI_HASH_SAME_HASH
   h->val2 = 0;
-  h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
-  h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
-  h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
   for (i = 0; i < 65536; i++) {
     h->head2[i] = -1;
   }
@@ -129,7 +138,6 @@ void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
 
 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
                 ZopfliHash* h) {
-  (void)end;
   UpdateHashValue(h, array[pos + 0]);
-  UpdateHashValue(h, array[pos + 1]);
+  if (pos + 1 < end) UpdateHashValue(h, array[pos + 1]);
 }

data/vendor/zopfli/{hash.h → src/zopfli/hash.h}

@@ -27,16 +27,16 @@ The hash for ZopfliFindLongestMatch of lz77.c.
 #include "util.h"
 
 typedef struct ZopfliHash {
-  int* head;  /* Hash value to index of its most recent occurance. */
-  unsigned short* prev;  /* Index to index of prev. occurance of same hash. */
+  int* head;  /* Hash value to index of its most recent occurrence. */
+  unsigned short* prev;  /* Index to index of prev. occurrence of same hash. */
   int* hashval;  /* Index to hash value at this index. */
   int val;  /* Current hash value. */
 
 #ifdef ZOPFLI_HASH_SAME_HASH
   /* Fields with similar purpose as the above hash, but for the second hash with
   a value that is calculated differently.  */
-  int* head2;  /* Hash value to index of its most recent occurance. */
-  unsigned short* prev2;  /* Index to index of prev. occurance of same hash. */
+  int* head2;  /* Hash value to index of its most recent occurrence. */
+  unsigned short* prev2;  /* Index to index of prev. occurrence of same hash. */
   int* hashval2;  /* Index to hash value at this index. */
   int val2;  /* Current hash value. */
 #endif
@@ -46,10 +46,13 @@ typedef struct ZopfliHash {
 #endif
 } ZopfliHash;
 
-/* Allocates and initializes all fields of ZopfliHash. */
-void ZopfliInitHash(size_t window_size, ZopfliHash* h);
+/* Allocates ZopfliHash memory. */
+void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
 
-/* Frees all fields of ZopfliHash. */
+/* Resets all fields of ZopfliHash. */
+void ZopfliResetHash(size_t window_size, ZopfliHash* h);
+
+/* Frees ZopfliHash memory. */
 void ZopfliCleanHash(ZopfliHash* h);
 
 /*

data/vendor/zopfli/{katajainen.c → src/zopfli/katajainen.c}

@@ -26,6 +26,7 @@ Jyrki Katajainen, Alistair Moffat, Andrew Turpin".
 #include "katajainen.h"
 #include <assert.h>
 #include <stdlib.h>
+#include <limits.h>
 
 typedef struct Node Node;
 
@@ -36,16 +37,13 @@ struct Node {
   size_t weight;  /* Total weight (symbol count) of this chain. */
   Node* tail;  /* Previous node(s) of this chain, or 0 if none. */
   int count;  /* Leaf symbol index, or number of leaves before this chain. */
-  char inuse;  /* Tracking for garbage collection. */
 };
 
 /*
 Memory pool for nodes.
 */
 typedef struct NodePool {
-  Node* nodes;  /* The pool. */
-  Node* next;  /* Pointer to a possibly free node in the pool. */
-  int size;  /* Size of the memory pool. */
+  Node* next;  /* Pointer to a free node in the pool. */
 } NodePool;
 
 /*
@@ -55,40 +53,8 @@ static void InitNode(size_t weight, int count, Node* tail, Node* node) {
   node->weight = weight;
   node->count = count;
   node->tail = tail;
-  node->inuse = 1;
 }
 
-/*
-Finds a free location in the memory pool. Performs garbage collection if needed.
-lists: If given, used to mark in-use nodes during garbage collection.
-maxbits: Size of lists.
-pool: Memory pool to get free node from.
-*/
-static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) {
-  for (;;) {
-    if (pool->next >= &pool->nodes[pool->size]) {
-      /* Garbage collection. */
-      int i;
-      for (i = 0; i < pool->size; i++) {
-        pool->nodes[i].inuse = 0;
-      }
-      if (lists) {
-        for (i = 0; i < maxbits * 2; i++) {
-          Node* node;
-          for (node = lists[i / 2][i % 2]; node; node = node->tail) {
-            node->inuse = 1;
-          }
-        }
-      }
-      pool->next = &pool->nodes[0];
-    }
-    if (!pool->next->inuse) break;  /* Found one. */
-    pool->next++;
-  }
-  return pool->next++;
-}
-
-
 /*
 Performs a Boundary Package-Merge step. Puts a new chain in the given list. The
 new chain is, depending on the weights, a leaf or a combination of two chains
@@ -99,18 +65,16 @@ leaves: The leaves, one per symbol.
 numsymbols: Number of leaves.
 pool: the node memory pool.
 index: The index of the list in which a new chain or leaf is required.
-final: Whether this is the last time this function is called. If it is then it
-  is no more needed to recursively call self.
 */
-static void BoundaryPM(Node* (*lists)[2], int maxbits,
-    Node* leaves, int numsymbols, NodePool* pool, int index, char final) {
+static void BoundaryPM(Node* (*lists)[2], Node* leaves, int numsymbols,
+                       NodePool* pool, int index) {
   Node* newchain;
   Node* oldchain;
   int lastcount = lists[index][1]->count;  /* Count of last chain of list. */
 
   if (index == 0 && lastcount >= numsymbols) return;
 
-  newchain = GetFreeNode(lists, maxbits, pool);
+  newchain = pool->next++;
   oldchain = lists[index][1];
 
   /* These are set up before the recursive calls below, so that there is a list
@@ -129,15 +93,31 @@ static void BoundaryPM(Node* (*lists)[2], int maxbits,
           newchain);
     } else {
       InitNode(sum, lastcount, lists[index - 1][1], newchain);
-      if (!final) {
-        /* Two lookahead chains of previous list used up, create new ones. */
-        BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
-        BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
-      }
+      /* Two lookahead chains of previous list used up, create new ones. */
+      BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
+      BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
     }
   }
 }
 
+static void BoundaryPMFinal(Node* (*lists)[2],
+    Node* leaves, int numsymbols, NodePool* pool, int index) {
+  int lastcount = lists[index][1]->count;  /* Count of last chain of list. */
+
+  size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
+
+  if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
+    Node* newchain = pool->next;
+    Node* oldchain = lists[index][1]->tail;
+
+    lists[index][1] = newchain;
+    newchain->count = lastcount + 1;
+    newchain->tail = oldchain;
+  } else {
+    lists[index][1]->tail = lists[index - 1][1];
+  }
+}
+
 /*
 Initializes each list with as lookahead chains the two leaves with lowest
 weights.
@@ -145,8 +125,8 @@ weights.
 static void InitLists(
     NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) {
   int i;
-  Node* node0 = GetFreeNode(0, maxbits, pool);
-  Node* node1 = GetFreeNode(0, maxbits, pool);
+  Node* node0 = pool->next++;
+  Node* node1 = pool->next++;
   InitNode(leaves[0].weight, 1, 0, node0);
   InitNode(leaves[1].weight, 2, 0, node1);
   for (i = 0; i < maxbits; i++) {
@@ -161,12 +141,24 @@ last chain of the last list contains the amount of active leaves in each list.
 chain: Chain to extract the bit length from (last chain from last list).
 */
 static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) {
+  int counts[16] = {0};
+  unsigned end = 16;
+  unsigned ptr = 15;
+  unsigned value = 1;
   Node* node;
+  int val;
+
   for (node = chain; node; node = node->tail) {
-    int i;
-    for (i = 0; i < node->count; i++) {
-      bitlengths[leaves[i].count]++;
+    counts[--end] = node->count;
+  }
+
+  val = counts[15];
+  while (ptr >= end) {
+    for (; val > counts[ptr - 1]; val--) {
+      bitlengths[leaves[val - 1].count] = value;
     }
+    ptr--;
+    value++;
   }
 }
 
@@ -183,6 +175,7 @@ int ZopfliLengthLimitedCodeLengths(
   int i;
   int numsymbols = 0;  /* Amount of symbols with frequency > 0. */
   int numBoundaryPMRuns;
+  Node* nodes;
 
   /* Array of lists of chains. Each list requires only two lookahead chains at
   a time, so each list is a array of two Node*'s. */
@@ -219,33 +212,51 @@ int ZopfliLengthLimitedCodeLengths(
     free(leaves);
     return 0;  /* Only one symbol, give it bitlength 1, not 0. OK. */
   }
+  if (numsymbols == 2) {
+    bitlengths[leaves[0].count]++;
+    bitlengths[leaves[1].count]++;
+    free(leaves);
+    return 0;
+  }
 
-  /* Sort the leaves from lightest to heaviest. */
+  /* Sort the leaves from lightest to heaviest. Add count into the same
+  variable for stable sorting. */
+  for (i = 0; i < numsymbols; i++) {
+    if (leaves[i].weight >=
+        ((size_t)1 << (sizeof(leaves[0].weight) * CHAR_BIT - 9))) {
+      free(leaves);
+      return 1;  /* Error, we need 9 bits for the count. */
+    }
+    leaves[i].weight = (leaves[i].weight << 9) | leaves[i].count;
+  }
   qsort(leaves, numsymbols, sizeof(Node), LeafComparator);
+  for (i = 0; i < numsymbols; i++) {
+    leaves[i].weight >>= 9;
+  }
 
-  /* Initialize node memory pool. */
-  pool.size = 2 * maxbits * (maxbits + 1);
-  pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes));
-  pool.next = pool.nodes;
-  for (i = 0; i < pool.size; i++) {
-    pool.nodes[i].inuse = 0;
+  if (numsymbols - 1 < maxbits) {
+    maxbits = numsymbols - 1;
   }
 
+  /* Initialize node memory pool. */
+  nodes = (Node*)malloc(maxbits * 2 * numsymbols * sizeof(Node));
+  pool.next = nodes;
+
   lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists));
   InitLists(&pool, leaves, maxbits, lists);
 
   /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two
   are already created in the initialization. Each BoundaryPM run creates one. */
   numBoundaryPMRuns = 2 * numsymbols - 4;
-  for (i = 0; i < numBoundaryPMRuns; i++) {
-    char final = i == numBoundaryPMRuns - 1;
-    BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final);
+  for (i = 0; i < numBoundaryPMRuns - 1; i++) {
+    BoundaryPM(lists, leaves, numsymbols, &pool, maxbits - 1);
   }
+  BoundaryPMFinal(lists, leaves, numsymbols, &pool, maxbits - 1);
 
   ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths);
 
   free(lists);
   free(leaves);
-  free(pool.nodes);
+  free(nodes);
   return 0;  /* OK. */
 }