zopfli 0.0.5 → 0.0.7

data/vendor/zopfli/src/zopfli/deflate.h

@@ -25,6 +25,7 @@ Functions to compress according to the DEFLATE specification, using the
 "squeeze" LZ77 compression backend.
 */
 
+#include "lz77.h"
 #include "zopfli.h"
 
 #ifdef __cplusplus
@@ -75,10 +76,15 @@ dists: ll77 distances
 lstart: start of block
 lend: end of block (not inclusive)
 */
-double ZopfliCalculateBlockSize(const unsigned short* litlens,
-                                const unsigned short* dists,
+double ZopfliCalculateBlockSize(const ZopfliLZ77Store* lz77,
                                 size_t lstart, size_t lend, int btype);
 
+/*
+Calculates block size in bits, automatically using the best btype.
+*/
+double ZopfliCalculateBlockSizeAutoType(const ZopfliLZ77Store* lz77,
+                                        size_t lstart, size_t lend);
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif

data/vendor/zopfli/src/zopfli/gzip_container.c

@@ -24,56 +24,63 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
 
 #include "deflate.h"
 
-/* Table of CRCs of all 8-bit messages. */
-static unsigned long crc_table[256];
-
-/* Flag: has the table been computed? Initially false. */
-static int crc_table_computed = 0;
-
-/* Makes the table for a fast CRC. */
-static void MakeCRCTable() {
-  unsigned long c;
-  int n, k;
-  for (n = 0; n < 256; n++) {
-    c = (unsigned long) n;
-    for (k = 0; k < 8; k++) {
-      if (c & 1) {
-        c = 0xedb88320L ^ (c >> 1);
-      } else {
-        c = c >> 1;
-      }
-    }
-    crc_table[n] = c;
+/* CRC polynomial: 0xedb88320 */
+static const unsigned long crc32_table[256] = {
+           0u, 1996959894u, 3993919788u, 2567524794u,  124634137u, 1886057615u,
+  3915621685u, 2657392035u,  249268274u, 2044508324u, 3772115230u, 2547177864u,
+   162941995u, 2125561021u, 3887607047u, 2428444049u,  498536548u, 1789927666u,
+  4089016648u, 2227061214u,  450548861u, 1843258603u, 4107580753u, 2211677639u,
+   325883990u, 1684777152u, 4251122042u, 2321926636u,  335633487u, 1661365465u,
+  4195302755u, 2366115317u,  997073096u, 1281953886u, 3579855332u, 2724688242u,
+  1006888145u, 1258607687u, 3524101629u, 2768942443u,  901097722u, 1119000684u,
+  3686517206u, 2898065728u,  853044451u, 1172266101u, 3705015759u, 2882616665u,
+   651767980u, 1373503546u, 3369554304u, 3218104598u,  565507253u, 1454621731u,
+  3485111705u, 3099436303u,  671266974u, 1594198024u, 3322730930u, 2970347812u,
+   795835527u, 1483230225u, 3244367275u, 3060149565u, 1994146192u,   31158534u,
+  2563907772u, 4023717930u, 1907459465u,  112637215u, 2680153253u, 3904427059u,
+  2013776290u,  251722036u, 2517215374u, 3775830040u, 2137656763u,  141376813u,
+  2439277719u, 3865271297u, 1802195444u,  476864866u, 2238001368u, 4066508878u,
+  1812370925u,  453092731u, 2181625025u, 4111451223u, 1706088902u,  314042704u,
+  2344532202u, 4240017532u, 1658658271u,  366619977u, 2362670323u, 4224994405u,
+  1303535960u,  984961486u, 2747007092u, 3569037538u, 1256170817u, 1037604311u,
+  2765210733u, 3554079995u, 1131014506u,  879679996u, 2909243462u, 3663771856u,
+  1141124467u,  855842277u, 2852801631u, 3708648649u, 1342533948u,  654459306u,
+  3188396048u, 3373015174u, 1466479909u,  544179635u, 3110523913u, 3462522015u,
+  1591671054u,  702138776u, 2966460450u, 3352799412u, 1504918807u,  783551873u,
+  3082640443u, 3233442989u, 3988292384u, 2596254646u,   62317068u, 1957810842u,
+  3939845945u, 2647816111u,   81470997u, 1943803523u, 3814918930u, 2489596804u,
+   225274430u, 2053790376u, 3826175755u, 2466906013u,  167816743u, 2097651377u,
+  4027552580u, 2265490386u,  503444072u, 1762050814u, 4150417245u, 2154129355u,
+   426522225u, 1852507879u, 4275313526u, 2312317920u,  282753626u, 1742555852u,
+  4189708143u, 2394877945u,  397917763u, 1622183637u, 3604390888u, 2714866558u,
+   953729732u, 1340076626u, 3518719985u, 2797360999u, 1068828381u, 1219638859u,
+  3624741850u, 2936675148u,  906185462u, 1090812512u, 3747672003u, 2825379669u,
+   829329135u, 1181335161u, 3412177804u, 3160834842u,  628085408u, 1382605366u,
+  3423369109u, 3138078467u,  570562233u, 1426400815u, 3317316542u, 2998733608u,
+   733239954u, 1555261956u, 3268935591u, 3050360625u,  752459403u, 1541320221u,
+  2607071920u, 3965973030u, 1969922972u,   40735498u, 2617837225u, 3943577151u,
+  1913087877u,   83908371u, 2512341634u, 3803740692u, 2075208622u,  213261112u,
+  2463272603u, 3855990285u, 2094854071u,  198958881u, 2262029012u, 4057260610u,
+  1759359992u,  534414190u, 2176718541u, 4139329115u, 1873836001u,  414664567u,
+  2282248934u, 4279200368u, 1711684554u,  285281116u, 2405801727u, 4167216745u,
+  1634467795u,  376229701u, 2685067896u, 3608007406u, 1308918612u,  956543938u,
+  2808555105u, 3495958263u, 1231636301u, 1047427035u, 2932959818u, 3654703836u,
+  1088359270u,  936918000u, 2847714899u, 3736837829u, 1202900863u,  817233897u,
+  3183342108u, 3401237130u, 1404277552u,  615818150u, 3134207493u, 3453421203u,
+  1423857449u,  601450431u, 3009837614u, 3294710456u, 1567103746u,  711928724u,
+  3020668471u, 3272380065u, 1510334235u,  755167117u
+};
+
+/* Returns the CRC32 */
+static unsigned long CRC(const unsigned char* data, size_t size) {
+  unsigned long result = 0xffffffffu;
+  for (; size > 0; size--) {
+    result = crc32_table[(result ^ *(data++)) & 0xff] ^ (result >> 8);
   }
-  crc_table_computed = 1;
+  return result ^ 0xffffffffu;
 }
 
-
-/*
-Updates a running crc with the bytes buf[0..len-1] and returns
-the updated crc. The crc should be initialized to zero.
-*/
-static unsigned long UpdateCRC(unsigned long crc,
-                               const unsigned char *buf, size_t len) {
-  unsigned long c = crc ^ 0xffffffffL;
-  unsigned n;
-
-  if (!crc_table_computed)
-    MakeCRCTable();
-  for (n = 0; n < len; n++) {
-    c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
-  }
-  return c ^ 0xffffffffL;
-}
-
-/* Returns the CRC of the bytes buf[0..len-1]. */
-static unsigned long CRC(const unsigned char* buf, int len) {
-  return UpdateCRC(0L, buf, len);
-}
-
-/*
-Compresses the data according to the gzip specification.
-*/
+/* Compresses the data according to the gzip specification, RFC 1952. */
 void ZopfliGzipCompress(const ZopfliOptions* options,
                         const unsigned char* in, size_t insize,
                         unsigned char** out, size_t* outsize) {

data/vendor/zopfli/src/zopfli/hash.c

@@ -26,13 +26,26 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
 #define HASH_SHIFT 5
 #define HASH_MASK 32767
 
-void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
-  size_t i;
-
-  h->val = 0;
+void ZopfliAllocHash(size_t window_size, ZopfliHash* h) {
   h->head = (int*)malloc(sizeof(*h->head) * 65536);
   h->prev = (unsigned short*)malloc(sizeof(*h->prev) * window_size);
   h->hashval = (int*)malloc(sizeof(*h->hashval) * window_size);
+
+#ifdef ZOPFLI_HASH_SAME
+  h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
+#endif
+
+#ifdef ZOPFLI_HASH_SAME_HASH
+  h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
+  h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
+  h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
+#endif
+}
+
+void ZopfliResetHash(size_t window_size, ZopfliHash* h) {
+  size_t i;
+
+  h->val = 0;
   for (i = 0; i < 65536; i++) {
     h->head[i] = -1;  /* -1 indicates no head so far. */
   }
@@ -42,7 +55,6 @@ void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
   }
 
 #ifdef ZOPFLI_HASH_SAME
-  h->same = (unsigned short*)malloc(sizeof(*h->same) * window_size);
   for (i = 0; i < window_size; i++) {
     h->same[i] = 0;
   }
@@ -50,9 +62,6 @@ void ZopfliInitHash(size_t window_size, ZopfliHash* h) {
 
 #ifdef ZOPFLI_HASH_SAME_HASH
   h->val2 = 0;
-  h->head2 = (int*)malloc(sizeof(*h->head2) * 65536);
-  h->prev2 = (unsigned short*)malloc(sizeof(*h->prev2) * window_size);
-  h->hashval2 = (int*)malloc(sizeof(*h->hashval2) * window_size);
   for (i = 0; i < 65536; i++) {
     h->head2[i] = -1;
   }
@@ -129,7 +138,6 @@ void ZopfliUpdateHash(const unsigned char* array, size_t pos, size_t end,
 
 void ZopfliWarmupHash(const unsigned char* array, size_t pos, size_t end,
                 ZopfliHash* h) {
-  (void)end;
   UpdateHashValue(h, array[pos + 0]);
-  UpdateHashValue(h, array[pos + 1]);
+  if (pos + 1 < end) UpdateHashValue(h, array[pos + 1]);
 }

data/vendor/zopfli/src/zopfli/hash.h

@@ -46,10 +46,13 @@ typedef struct ZopfliHash {
 #endif
 } ZopfliHash;
 
-/* Allocates and initializes all fields of ZopfliHash. */
-void ZopfliInitHash(size_t window_size, ZopfliHash* h);
+/* Allocates ZopfliHash memory. */
+void ZopfliAllocHash(size_t window_size, ZopfliHash* h);
 
-/* Frees all fields of ZopfliHash. */
+/* Resets all fields of ZopfliHash. */
+void ZopfliResetHash(size_t window_size, ZopfliHash* h);
+
+/* Frees ZopfliHash memory. */
 void ZopfliCleanHash(ZopfliHash* h);
 
 /*

data/vendor/zopfli/src/zopfli/katajainen.c

@@ -26,6 +26,7 @@ Jyrki Katajainen, Alistair Moffat, Andrew Turpin".
 #include "katajainen.h"
 #include <assert.h>
 #include <stdlib.h>
+#include <limits.h>
 
 typedef struct Node Node;
 
@@ -36,16 +37,13 @@ struct Node {
   size_t weight;  /* Total weight (symbol count) of this chain. */
   Node* tail;  /* Previous node(s) of this chain, or 0 if none. */
   int count;  /* Leaf symbol index, or number of leaves before this chain. */
-  char inuse;  /* Tracking for garbage collection. */
 };
 
 /*
 Memory pool for nodes.
 */
 typedef struct NodePool {
-  Node* nodes;  /* The pool. */
-  Node* next;  /* Pointer to a possibly free node in the pool. */
-  int size;  /* Size of the memory pool. */
+  Node* next;  /* Pointer to a free node in the pool. */
 } NodePool;
 
 /*
@@ -55,40 +53,8 @@ static void InitNode(size_t weight, int count, Node* tail, Node* node) {
   node->weight = weight;
   node->count = count;
   node->tail = tail;
-  node->inuse = 1;
 }
 
-/*
-Finds a free location in the memory pool. Performs garbage collection if needed.
-lists: If given, used to mark in-use nodes during garbage collection.
-maxbits: Size of lists.
-pool: Memory pool to get free node from.
-*/
-static Node* GetFreeNode(Node* (*lists)[2], int maxbits, NodePool* pool) {
-  for (;;) {
-    if (pool->next >= &pool->nodes[pool->size]) {
-      /* Garbage collection. */
-      int i;
-      for (i = 0; i < pool->size; i++) {
-        pool->nodes[i].inuse = 0;
-      }
-      if (lists) {
-        for (i = 0; i < maxbits * 2; i++) {
-          Node* node;
-          for (node = lists[i / 2][i % 2]; node; node = node->tail) {
-            node->inuse = 1;
-          }
-        }
-      }
-      pool->next = &pool->nodes[0];
-    }
-    if (!pool->next->inuse) break;  /* Found one. */
-    pool->next++;
-  }
-  return pool->next++;
-}
-
-
 /*
 Performs a Boundary Package-Merge step. Puts a new chain in the given list. The
 new chain is, depending on the weights, a leaf or a combination of two chains
@@ -99,18 +65,16 @@ leaves: The leaves, one per symbol.
 numsymbols: Number of leaves.
 pool: the node memory pool.
 index: The index of the list in which a new chain or leaf is required.
-final: Whether this is the last time this function is called. If it is then it
-  is no more needed to recursively call self.
 */
-static void BoundaryPM(Node* (*lists)[2], int maxbits,
-    Node* leaves, int numsymbols, NodePool* pool, int index, char final) {
+static void BoundaryPM(Node* (*lists)[2], Node* leaves, int numsymbols,
+                       NodePool* pool, int index) {
   Node* newchain;
   Node* oldchain;
   int lastcount = lists[index][1]->count;  /* Count of last chain of list. */
 
   if (index == 0 && lastcount >= numsymbols) return;
 
-  newchain = GetFreeNode(lists, maxbits, pool);
+  newchain = pool->next++;
   oldchain = lists[index][1];
 
   /* These are set up before the recursive calls below, so that there is a list
@@ -129,15 +93,31 @@ static void BoundaryPM(Node* (*lists)[2], int maxbits,
           newchain);
     } else {
       InitNode(sum, lastcount, lists[index - 1][1], newchain);
-      if (!final) {
-        /* Two lookahead chains of previous list used up, create new ones. */
-        BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
-        BoundaryPM(lists, maxbits, leaves, numsymbols, pool, index - 1, 0);
-      }
+      /* Two lookahead chains of previous list used up, create new ones. */
+      BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
+      BoundaryPM(lists, leaves, numsymbols, pool, index - 1);
     }
   }
 }
 
+static void BoundaryPMFinal(Node* (*lists)[2],
+    Node* leaves, int numsymbols, NodePool* pool, int index) {
+  int lastcount = lists[index][1]->count;  /* Count of last chain of list. */
+
+  size_t sum = lists[index - 1][0]->weight + lists[index - 1][1]->weight;
+
+  if (lastcount < numsymbols && sum > leaves[lastcount].weight) {
+    Node* newchain = pool->next;
+    Node* oldchain = lists[index][1]->tail;
+
+    lists[index][1] = newchain;
+    newchain->count = lastcount + 1;
+    newchain->tail = oldchain;
+  } else {
+    lists[index][1]->tail = lists[index - 1][1];
+  }
+}
+
 /*
 Initializes each list with as lookahead chains the two leaves with lowest
 weights.
@@ -145,8 +125,8 @@ weights.
 static void InitLists(
     NodePool* pool, const Node* leaves, int maxbits, Node* (*lists)[2]) {
   int i;
-  Node* node0 = GetFreeNode(0, maxbits, pool);
-  Node* node1 = GetFreeNode(0, maxbits, pool);
+  Node* node0 = pool->next++;
+  Node* node1 = pool->next++;
   InitNode(leaves[0].weight, 1, 0, node0);
   InitNode(leaves[1].weight, 2, 0, node1);
   for (i = 0; i < maxbits; i++) {
@@ -161,12 +141,24 @@ last chain of the last list contains the amount of active leaves in each list.
 chain: Chain to extract the bit length from (last chain from last list).
 */
 static void ExtractBitLengths(Node* chain, Node* leaves, unsigned* bitlengths) {
+  int counts[16] = {0};
+  unsigned end = 16;
+  unsigned ptr = 15;
+  unsigned value = 1;
   Node* node;
+  int val;
+
   for (node = chain; node; node = node->tail) {
-    int i;
-    for (i = 0; i < node->count; i++) {
-      bitlengths[leaves[i].count]++;
+    counts[--end] = node->count;
+  }
+
+  val = counts[15];
+  while (ptr >= end) {
+    for (; val > counts[ptr - 1]; val--) {
+      bitlengths[leaves[val - 1].count] = value;
     }
+    ptr--;
+    value++;
   }
 }
 
@@ -183,6 +175,7 @@ int ZopfliLengthLimitedCodeLengths(
   int i;
   int numsymbols = 0;  /* Amount of symbols with frequency > 0. */
   int numBoundaryPMRuns;
+  Node* nodes;
 
   /* Array of lists of chains. Each list requires only two lookahead chains at
   a time, so each list is a array of two Node*'s. */
@@ -219,33 +212,51 @@ int ZopfliLengthLimitedCodeLengths(
     free(leaves);
     return 0;  /* Only one symbol, give it bitlength 1, not 0. OK. */
   }
+  if (numsymbols == 2) {
+    bitlengths[leaves[0].count]++;
+    bitlengths[leaves[1].count]++;
+    free(leaves);
+    return 0;
+  }
 
-  /* Sort the leaves from lightest to heaviest. */
+  /* Sort the leaves from lightest to heaviest. Add count into the same
+  variable for stable sorting. */
+  for (i = 0; i < numsymbols; i++) {
+    if (leaves[i].weight >=
+        ((size_t)1 << (sizeof(leaves[0].weight) * CHAR_BIT - 9))) {
+      free(leaves);
+      return 1;  /* Error, we need 9 bits for the count. */
+    }
+    leaves[i].weight = (leaves[i].weight << 9) | leaves[i].count;
+  }
   qsort(leaves, numsymbols, sizeof(Node), LeafComparator);
+  for (i = 0; i < numsymbols; i++) {
+    leaves[i].weight >>= 9;
+  }
 
-  /* Initialize node memory pool. */
-  pool.size = 2 * maxbits * (maxbits + 1);
-  pool.nodes = (Node*)malloc(pool.size * sizeof(*pool.nodes));
-  pool.next = pool.nodes;
-  for (i = 0; i < pool.size; i++) {
-    pool.nodes[i].inuse = 0;
+  if (numsymbols - 1 < maxbits) {
+    maxbits = numsymbols - 1;
   }
 
+  /* Initialize node memory pool. */
+  nodes = (Node*)malloc(maxbits * 2 * numsymbols * sizeof(Node));
+  pool.next = nodes;
+
   lists = (Node* (*)[2])malloc(maxbits * sizeof(*lists));
   InitLists(&pool, leaves, maxbits, lists);
 
   /* In the last list, 2 * numsymbols - 2 active chains need to be created. Two
   are already created in the initialization. Each BoundaryPM run creates one. */
   numBoundaryPMRuns = 2 * numsymbols - 4;
-  for (i = 0; i < numBoundaryPMRuns; i++) {
-    char final = i == numBoundaryPMRuns - 1;
-    BoundaryPM(lists, maxbits, leaves, numsymbols, &pool, maxbits - 1, final);
+  for (i = 0; i < numBoundaryPMRuns - 1; i++) {
+    BoundaryPM(lists, leaves, numsymbols, &pool, maxbits - 1);
   }
+  BoundaryPMFinal(lists, leaves, numsymbols, &pool, maxbits - 1);
 
   ExtractBitLengths(lists[maxbits - 1][1], leaves, bitlengths);
 
   free(lists);
   free(leaves);
-  free(pool.nodes);
+  free(nodes);
   return 0;  /* OK. */
 }

data/vendor/zopfli/src/zopfli/lz77.c

@@ -18,37 +18,76 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
 */
 
 #include "lz77.h"
+#include "symbols.h"
 #include "util.h"
 
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 
-void ZopfliInitLZ77Store(ZopfliLZ77Store* store) {
+void ZopfliInitLZ77Store(const unsigned char* data, ZopfliLZ77Store* store) {
   store->size = 0;
   store->litlens = 0;
   store->dists = 0;
+  store->pos = 0;
+  store->data = data;
+  store->ll_symbol = 0;
+  store->d_symbol = 0;
+  store->ll_counts = 0;
+  store->d_counts = 0;
 }
 
 void ZopfliCleanLZ77Store(ZopfliLZ77Store* store) {
   free(store->litlens);
   free(store->dists);
+  free(store->pos);
+  free(store->ll_symbol);
+  free(store->d_symbol);
+  free(store->ll_counts);
+  free(store->d_counts);
+}
+
+static size_t CeilDiv(size_t a, size_t b) {
+  return (a + b - 1) / b;
 }
 
 void ZopfliCopyLZ77Store(
     const ZopfliLZ77Store* source, ZopfliLZ77Store* dest) {
   size_t i;
+  size_t llsize = ZOPFLI_NUM_LL * CeilDiv(source->size, ZOPFLI_NUM_LL);
+  size_t dsize = ZOPFLI_NUM_D * CeilDiv(source->size, ZOPFLI_NUM_D);
   ZopfliCleanLZ77Store(dest);
+  ZopfliInitLZ77Store(source->data, dest);
   dest->litlens =
       (unsigned short*)malloc(sizeof(*dest->litlens) * source->size);
   dest->dists = (unsigned short*)malloc(sizeof(*dest->dists) * source->size);
-
-  if (!dest->litlens || !dest->dists) exit(-1); /* Allocation failed. */
+  dest->pos = (size_t*)malloc(sizeof(*dest->pos) * source->size);
+  dest->ll_symbol =
+      (unsigned short*)malloc(sizeof(*dest->ll_symbol) * source->size);
+  dest->d_symbol =
+      (unsigned short*)malloc(sizeof(*dest->d_symbol) * source->size);
+  dest->ll_counts = (size_t*)malloc(sizeof(*dest->ll_counts) * llsize);
+  dest->d_counts = (size_t*)malloc(sizeof(*dest->d_counts) * dsize);
+
+  /* Allocation failed. */
+  if (!dest->litlens || !dest->dists) exit(-1);
+  if (!dest->pos) exit(-1);
+  if (!dest->ll_symbol || !dest->d_symbol) exit(-1);
+  if (!dest->ll_counts || !dest->d_counts) exit(-1);
 
   dest->size = source->size;
   for (i = 0; i < source->size; i++) {
     dest->litlens[i] = source->litlens[i];
     dest->dists[i] = source->dists[i];
+    dest->pos[i] = source->pos[i];
+    dest->ll_symbol[i] = source->ll_symbol[i];
+    dest->d_symbol[i] = source->d_symbol[i];
+  }
+  for (i = 0; i < llsize; i++) {
+    dest->ll_counts[i] = source->ll_counts[i];
+  }
+  for (i = 0; i < dsize; i++) {
+    dest->d_counts[i] = source->d_counts[i];
   }
 }
 
@@ -57,10 +96,149 @@ Appends the length and distance to the LZ77 arrays of the ZopfliLZ77Store.
 context must be a ZopfliLZ77Store*.
 */
 void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
-                           ZopfliLZ77Store* store) {
-  size_t size2 = store->size;  /* Needed for using ZOPFLI_APPEND_DATA twice. */
+                           size_t pos, ZopfliLZ77Store* store) {
+  size_t i;
+  /* Needed for using ZOPFLI_APPEND_DATA multiple times. */
+  size_t origsize = store->size;
+  size_t llstart = ZOPFLI_NUM_LL * (origsize / ZOPFLI_NUM_LL);
+  size_t dstart = ZOPFLI_NUM_D * (origsize / ZOPFLI_NUM_D);
+
+  /* Everytime the index wraps around, a new cumulative histogram is made: we're
+  keeping one histogram value per LZ77 symbol rather than a full histogram for
+  each to save memory. */
+  if (origsize % ZOPFLI_NUM_LL == 0) {
+    size_t llsize = origsize;
+    for (i = 0; i < ZOPFLI_NUM_LL; i++) {
+      ZOPFLI_APPEND_DATA(
+          origsize == 0 ? 0 : store->ll_counts[origsize - ZOPFLI_NUM_LL + i],
+          &store->ll_counts, &llsize);
+    }
+  }
+  if (origsize % ZOPFLI_NUM_D == 0) {
+    size_t dsize = origsize;
+    for (i = 0; i < ZOPFLI_NUM_D; i++) {
+      ZOPFLI_APPEND_DATA(
+          origsize == 0 ? 0 : store->d_counts[origsize - ZOPFLI_NUM_D + i],
+          &store->d_counts, &dsize);
+    }
+  }
+
   ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size);
-  ZOPFLI_APPEND_DATA(dist, &store->dists, &size2);
+  store->size = origsize;
+  ZOPFLI_APPEND_DATA(dist, &store->dists, &store->size);
+  store->size = origsize;
+  ZOPFLI_APPEND_DATA(pos, &store->pos, &store->size);
+  assert(length < 259);
+
+  if (dist == 0) {
+    store->size = origsize;
+    ZOPFLI_APPEND_DATA(length, &store->ll_symbol, &store->size);
+    store->size = origsize;
+    ZOPFLI_APPEND_DATA(0, &store->d_symbol, &store->size);
+    store->ll_counts[llstart + length]++;
+  } else {
+    store->size = origsize;
+    ZOPFLI_APPEND_DATA(ZopfliGetLengthSymbol(length),
+                       &store->ll_symbol, &store->size);
+    store->size = origsize;
+    ZOPFLI_APPEND_DATA(ZopfliGetDistSymbol(dist),
+                       &store->d_symbol, &store->size);
+    store->ll_counts[llstart + ZopfliGetLengthSymbol(length)]++;
+    store->d_counts[dstart + ZopfliGetDistSymbol(dist)]++;
+  }
+}
+
+void ZopfliAppendLZ77Store(const ZopfliLZ77Store* store,
+                           ZopfliLZ77Store* target) {
+  size_t i;
+  for (i = 0; i < store->size; i++) {
+    ZopfliStoreLitLenDist(store->litlens[i], store->dists[i],
+                          store->pos[i], target);
+  }
+}
+
+size_t ZopfliLZ77GetByteRange(const ZopfliLZ77Store* lz77,
+                              size_t lstart, size_t lend) {
+  size_t l = lend - 1;
+  if (lstart == lend) return 0;
+  return lz77->pos[l] + ((lz77->dists[l] == 0) ?
+      1 : lz77->litlens[l]) - lz77->pos[lstart];
+}
+
+static void ZopfliLZ77GetHistogramAt(const ZopfliLZ77Store* lz77, size_t lpos,
+                                     size_t* ll_counts, size_t* d_counts) {
+  /* The real histogram is created by using the histogram for this chunk, but
+  all superfluous values of this chunk subtracted. */
+  size_t llpos = ZOPFLI_NUM_LL * (lpos / ZOPFLI_NUM_LL);
+  size_t dpos = ZOPFLI_NUM_D * (lpos / ZOPFLI_NUM_D);
+  size_t i;
+  for (i = 0; i < ZOPFLI_NUM_LL; i++) {
+    ll_counts[i] = lz77->ll_counts[llpos + i];
+  }
+  for (i = lpos + 1; i < llpos + ZOPFLI_NUM_LL && i < lz77->size; i++) {
+    ll_counts[lz77->ll_symbol[i]]--;
+  }
+  for (i = 0; i < ZOPFLI_NUM_D; i++) {
+    d_counts[i] = lz77->d_counts[dpos + i];
+  }
+  for (i = lpos + 1; i < dpos + ZOPFLI_NUM_D && i < lz77->size; i++) {
+    if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]--;
+  }
+}
+
+void ZopfliLZ77GetHistogram(const ZopfliLZ77Store* lz77,
+                           size_t lstart, size_t lend,
+                           size_t* ll_counts, size_t* d_counts) {
+  size_t i;
+  if (lstart + ZOPFLI_NUM_LL * 3 > lend) {
+    memset(ll_counts, 0, sizeof(*ll_counts) * ZOPFLI_NUM_LL);
+    memset(d_counts, 0, sizeof(*d_counts) * ZOPFLI_NUM_D);
+    for (i = lstart; i < lend; i++) {
+      ll_counts[lz77->ll_symbol[i]]++;
+      if (lz77->dists[i] != 0) d_counts[lz77->d_symbol[i]]++;
+    }
+  } else {
+    /* Subtract the cumulative histograms at the end and the start to get the
+    histogram for this range. */
+    ZopfliLZ77GetHistogramAt(lz77, lend - 1, ll_counts, d_counts);
+    if (lstart > 0) {
+      size_t ll_counts2[ZOPFLI_NUM_LL];
+      size_t d_counts2[ZOPFLI_NUM_D];
+      ZopfliLZ77GetHistogramAt(lz77, lstart - 1, ll_counts2, d_counts2);
+
+      for (i = 0; i < ZOPFLI_NUM_LL; i++) {
+        ll_counts[i] -= ll_counts2[i];
+      }
+      for (i = 0; i < ZOPFLI_NUM_D; i++) {
+        d_counts[i] -= d_counts2[i];
+      }
+    }
+  }
+}
+
+void ZopfliInitBlockState(const ZopfliOptions* options,
+                          size_t blockstart, size_t blockend, int add_lmc,
+                          ZopfliBlockState* s) {
+  s->options = options;
+  s->blockstart = blockstart;
+  s->blockend = blockend;
+#ifdef ZOPFLI_LONGEST_MATCH_CACHE
+  if (add_lmc) {
+    s->lmc = (ZopfliLongestMatchCache*)malloc(sizeof(ZopfliLongestMatchCache));
+    ZopfliInitCache(blockend - blockstart, s->lmc);
+  } else {
+    s->lmc = 0;
+  }
+#endif
+}
+
+void ZopfliCleanBlockState(ZopfliBlockState* s) {
+#ifdef ZOPFLI_LONGEST_MATCH_CACHE
+  if (s->lmc) {
+    ZopfliCleanCache(s->lmc);
+    free(s->lmc);
+  }
+#endif
 }
 
 /*
@@ -365,7 +543,7 @@ void ZopfliFindLongestMatch(ZopfliBlockState* s, const ZopfliHash* h,
 
 void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
                       size_t instart, size_t inend,
-                      ZopfliLZ77Store* store) {
+                      ZopfliLZ77Store* store, ZopfliHash* h) {
   size_t i = 0, j;
   unsigned short leng;
   unsigned short dist;
@@ -374,9 +552,6 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
       ? instart - ZOPFLI_WINDOW_SIZE : 0;
   unsigned short dummysublen[259];
 
-  ZopfliHash hash;
-  ZopfliHash* h = &hash;
-
 #ifdef ZOPFLI_LAZY_MATCHING
   /* Lazy matching. */
   unsigned prev_length = 0;
@@ -387,7 +562,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
 
   if (instart == inend) return;
 
-  ZopfliInitHash(ZOPFLI_WINDOW_SIZE, h);
+  ZopfliResetHash(ZOPFLI_WINDOW_SIZE, h);
   ZopfliWarmupHash(in, windowstart, inend, h);
   for (i = windowstart; i < instart; i++) {
     ZopfliUpdateHash(in, i, inend, h);
@@ -406,7 +581,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
     if (match_available) {
       match_available = 0;
       if (lengthscore > prevlengthscore + 1) {
-        ZopfliStoreLitLenDist(in[i - 1], 0, store);
+        ZopfliStoreLitLenDist(in[i - 1], 0, i - 1, store);
         if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) {
           match_available = 1;
           prev_length = leng;
@@ -420,7 +595,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
         lengthscore = prevlengthscore;
         /* Add to output. */
         ZopfliVerifyLenDist(in, inend, i - 1, dist, leng);
-        ZopfliStoreLitLenDist(leng, dist, store);
+        ZopfliStoreLitLenDist(leng, dist, i - 1, store);
         for (j = 2; j < leng; j++) {
           assert(i < inend);
           i++;
@@ -441,10 +616,10 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
     /* Add to output. */
     if (lengthscore >= ZOPFLI_MIN_MATCH) {
       ZopfliVerifyLenDist(in, inend, i, dist, leng);
-      ZopfliStoreLitLenDist(leng, dist, store);
+      ZopfliStoreLitLenDist(leng, dist, i, store);
     } else {
       leng = 1;
-      ZopfliStoreLitLenDist(in[i], 0, store);
+      ZopfliStoreLitLenDist(in[i], 0, i, store);
     }
     for (j = 1; j < leng; j++) {
       assert(i < inend);
@@ -452,31 +627,4 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in,
       ZopfliUpdateHash(in, i, inend, h);
     }
   }
-
-  ZopfliCleanHash(h);
-}
-
-void ZopfliLZ77Counts(const unsigned short* litlens,
-                      const unsigned short* dists,
-                      size_t start, size_t end,
-                      size_t* ll_count, size_t* d_count) {
-  size_t i;
-
-  for (i = 0; i < 288; i++) {
-    ll_count[i] = 0;
-  }
-  for (i = 0; i < 32; i++) {
-    d_count[i] = 0;
-  }
-
-  for (i = start; i < end; i++) {
-    if (dists[i] == 0) {
-      ll_count[litlens[i]]++;
-    } else {
-      ll_count[ZopfliGetLengthSymbol(litlens[i])]++;
-      d_count[ZopfliGetDistSymbol(dists[i])]++;
-    }
-  }
-
-  ll_count[256] = 1;  /* End symbol. */
 }