zopfli 0.0.2 → 0.0.8

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MWNkODM0N2JmMDRjNGNiNmIyYzliYWVjYWRiOWJjZTdjM2FhMTczZg==
-  data.tar.gz: !binary |-
-    OWQ4YjNmMzBhNjc1NjRkOTUyYzk2Y2VkOTdmZmM3ODRmZTRjZjA4Nw==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    OTQxNThkMzU3OWRkMmE0YzU5MzE0YTFlNjBiZjJjMzg4NGVjYTAyMjNjZTY4
-    OTk1OGUyYTRjMzM1ODE5YjAxMjEyZTg1ZTY0MzhjYWRhYmVjNGVmNTgyZjhi
-    YzA2MWFiOGQyNjZmMmFiMWI4OGFiMjI5YmUxZDhiYmExYmVkNDM=
-  data.tar.gz: !binary |-
-    ZjM1N2E1MjA5NDViMmZkYmFhZWVjZWQzODE2ZDAxMmZkNDUyNWMzMGZlMTBl
-    MWM0OTA3OGIzOGViOGE0YTAyZGVjNTliZmVlMjJlMjBmZDhiODdiZjE1OWM3
-    M2VmODAzMWUyNDQ4YzJiMjcwNDk3Mzg2YjUxMzA0MTY0NzQzMWU=
+SHA256:
+  metadata.gz: 3121a61abacb0a102a03f4ec91ac9e6ee4a6132942d6e402bcfded04a25f39bd
+  data.tar.gz: 544aee94aaaa325abcaee491d8b052a59dbd6809794fb2cdd4dd3fd787c466dd
+SHA512:
+  metadata.gz: 5ff282d1b59f9bd4ab899cb1a16b63cadc64bb5a24c72b76289b136eadd8b50fcd14cb3a787169bc342ca1c6bb4ca5f405cc412eb3c13af6f4aa0c389cae2ec9
+  data.tar.gz: 24c803340ae499cf43da0d62b4fd916a0a05db1db07967461a7e65145810b70129a8063d4b972a1dad67ec8a6691b0ecf4d057390a78be771efae17983cae142

data/.gitmodules

@@ -1,3 +1,3 @@
 [submodule "vendor/zopfli"]
 	path = vendor/zopfli
-	url = https://code.google.com/p/zopfli/
+	url = https://github.com/google/zopfli.git

data/.travis.yml

@@ -0,0 +1,23 @@
+language: ruby
+sudo: false
+cache:
+  bundler: true
+  directories:
+  - "$HOME/.ccache"
+rvm:
+  - 2.3.3
+  - 2.4.0
+env:
+  global:
+    - PATH="/usr/lib/ccache:$PATH"
+before_install:
+  - travis_retry gem install bundler --no-document
+deploy:
+  provider: rubygems
+  api_key:
+    secure: Ix2H4oY1Ypo5mdYj9JTXufY/E7H/TWjAM54dtG2i/94yV8wMVINl+rhjnFso87LN6y0Tpt7rWy1OG1y3YPbwhBUPLbRXZ+xGjnsLA8/sUpEFODFk/QaYgOkCk2g3+xCgzzNvctd2u9fii/q9eIUts8BznegxtAYbat5a8dGvoVEkYAvHUX4lMRcEZVgMMpjT+xz3IGRe9rnW4umLd8P2mE6CEXAtrwFNSsjL1SVfKVOwwJ/UyiJfUnK6U/mDWdCDYWiPEE/CTz7PbrGGo4nAGoShLmkMUDsOamgECObT2rsNo42XzzrB4GAP+4PPQnias34BtJQLI9x2IEAUkM8vnKhSrtshp43S+ikU01W6rj4NMAvER4s9WNmRjsWgZQ/vEIHIwZXmdnkGvlYSiwRru+17YL2HCbrVp/YXi7X5HgBczuGOt3TZLB0xAKJG21qz+HdtP1nZTZe3EONDvWx9bM3XF2YjJ5gt3UNno8GsefTl71PFDubs1/LrVbkGgGwO6VTb4OgS2bEzntk17488BLTBo3l7bke5FTu8xQUbCawX2dqtOiukP45Ke8hRmgwWSMK+EH8YYcRcuZSbOXYJtGfpfC6nHtlArW+EjJ2Og+nWX68n7NFOIfGfCSX+V7cNgnKsqMjaesF+jEP+53CfVa2xnYxkZx7TgGrFyQEKr9s=
+  gem: zopfli
+  on:
+    tags: true
+    repo: miyucy/zopfli
+script: bundle exec rake && ./smoke.sh

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in zopfli.gemspec
 gemspec
+
+gem 'travis'

data/README.md

@@ -1,6 +1,6 @@
 # Zopfli
 
-see https://code.google.com/p/zopfli/
+see https://github.com/google/zopfli
 
 ## Installation
 
@@ -30,6 +30,11 @@ compressed_data = Zopfli.deflate string
 uncompressed_data = Zlib::Inflate.inflate compressed_data
 uncompressed_data == string
 # => true
+
+Zlib.deflate(File.read('LICENSE.txt'), Zlib::BEST_COMPRESSION).bytesize
+# => 628
+Zopfli.deflate(File.read('LICENSE.txt')).bytesize
+# => 601
 ```
 
 ## Contributing

data/Rakefile

@@ -1,25 +1,23 @@
 require "bundler/gem_tasks"
-require "rake/testtask"
+require "rspec/core/rake_task"
 require "rbconfig"
 
 DLEXT = RbConfig::CONFIG["DLEXT"]
 
-file "ext/zopfli.#{DLEXT}" => Dir.glob("ext/*{.rb,.c}") do
+file "ext/zopfli.#{DLEXT}" => Dir.glob("ext/*{.rb,.c,.h}") do
   Dir.chdir("ext") do
-     ruby "extconf.rb"
-     sh "make"
+    ruby "extconf.rb"
+    sh "make"
   end
   cp "ext/zopfli.#{DLEXT}", "lib"
 end
 
 task :clean do
   files = Dir["ext/*"] - ["ext/extconf.rb", "ext/zopfli.c"]
-  files+= ["ext/zopfli.#{DLEXT}", "lib/zopfli.#{DLEXT}"]
+  files += ["ext/zopfli.#{DLEXT}", "lib/zopfli.#{DLEXT}"]
   rm_rf(files) unless files.empty?
 end
 
-Rake::TestTask.new do |t|
-  t.warning = true
-  t.verbose = true
-end
-task :test => "ext/zopfli.#{DLEXT}"
+RSpec::Core::RakeTask.new(:spec)
+task :spec => "ext/zopfli.#{DLEXT}"
+task :default => :spec

data/ext/extconf.rb

@@ -9,7 +9,7 @@
 require "mkmf"
 
 dst = File.dirname File.expand_path __FILE__
-src = File.join dst, "..", "vendor", "zopfli"
+src = File.join dst, "..", "vendor", "zopfli", "src", "zopfli"
 
 %w(
 blocksplitter.c
@@ -28,6 +28,7 @@ lz77.c
 lz77.h
 squeeze.c
 squeeze.h
+symbols.h
 tree.c
 tree.h
 util.c

data/ext/zopfli.c

@@ -1,12 +1,11 @@
 #include "ruby.h"
+#include "ruby/thread.h"
 #include "zopfli.h"
 
 #define CSTR2SYM(x)    ID2SYM(rb_intern(x))
 #define DEFAULT_FORMAT ZOPFLI_FORMAT_ZLIB
 
-static VALUE rb_mZopfli;
-
-ZopfliFormat
+static ZopfliFormat
 zopfli_deflate_parse_options(ZopfliOptions *options, VALUE opts)
 {
     ZopfliFormat format;
@@ -50,35 +49,56 @@ zopfli_deflate_parse_options(ZopfliOptions *options, VALUE opts)
     return format;
 }
 
-VALUE
-zopfli_deflate(int argc, VALUE *argv, VALUE self)
-{
-    VALUE   in, out, opts;
+typedef struct {
     ZopfliOptions options;
     ZopfliFormat   format;
-    unsigned char    *tmp = NULL;
-    size_t        tmpsize = 0;
+    unsigned char     *in;
+    size_t         insize;
+    unsigned char    *out;
+    size_t        outsize;
+} zopfli_deflate_args_t;
+
+static void*
+zopfli_deflate_no_gvl(void* arg)
+{
+    zopfli_deflate_args_t *args = (zopfli_deflate_args_t*)arg;
 
-    ZopfliInitOptions(&options);
+    ZopfliCompress(&args->options, args->format,
+                   args->in, args->insize,
+                   &args->out, &args->outsize);
+
+    return arg;
+}
+
+static VALUE
+zopfli_deflate(int argc, VALUE *argv, VALUE self)
+{
+    zopfli_deflate_args_t args;
+    VALUE in, out, opts;
+
+    ZopfliInitOptions(&args.options);
 
     rb_scan_args(argc, argv, "11", &in, &opts);
 
     if (!NIL_P(opts)) {
-        format = zopfli_deflate_parse_options(&options, opts);
+        args.format = zopfli_deflate_parse_options(&args.options, opts);
     } else {
-        format = DEFAULT_FORMAT;
+        args.format = DEFAULT_FORMAT;
     }
 
     StringValue(in);
 
-    ZopfliCompress(&options,
-                   format,
-                   RSTRING_PTR(in), RSTRING_LEN(in),
-                   &tmp, &tmpsize);
+    args.in = (unsigned char*)RSTRING_PTR(in);
+    args.insize = RSTRING_LEN(in);
+
+    args.out = NULL;
+    args.outsize = 0;
+
+    rb_thread_call_without_gvl(zopfli_deflate_no_gvl, (void *)&args, NULL, NULL);
 
-    out = rb_str_new(tmp, tmpsize);
+    out = rb_str_new((const char*)args.out, args.outsize);
 
-    free(tmp);
+    free(args.out);
 
     return out;
 }
@@ -86,7 +106,6 @@ zopfli_deflate(int argc, VALUE *argv, VALUE self)
 void
 Init_zopfli()
 {
-    rb_mZopfli = rb_define_module("Zopfli");
+    VALUE rb_mZopfli = rb_define_module("Zopfli");
     rb_define_singleton_method(rb_mZopfli, "deflate", zopfli_deflate, -1);
-    rb_require("zopfli/version");
 }

data/lib/zopfli/version.rb

@@ -1,3 +1,3 @@
 module Zopfli
-  VERSION = "0.0.2"
+  VERSION = "0.0.8"
 end

data/smoke.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+gem list | grep zopfli && gem uninstall --force zopfli
+bundle exec rake clean build
+gem install --force --local --no-document "$(ls pkg/zopfli-*.gem)"
+cat <<EOF | ruby
+require 'zopfli'
+require 'zlib'
+abort if Zlib::Inflate.inflate(Zopfli.deflate(File.read('smoke.sh'))) != File.read('smoke.sh')
+EOF

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'zopfli'

data/spec/zopfli_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+require 'zlib'
+require 'stringio'
+require 'benchmark'
+require 'thread'
+
+RSpec.describe Zopfli do
+  let(:fixture) { File.read(File.expand_path('fixtures/alice29.txt', File.dirname(__FILE__))) }
+
+  it 'works' do
+    expect(Zopfli.deflate(fixture)).to eq Zopfli.deflate(fixture, format: :zlib)
+  end
+
+  it 'works (zlib format)' do
+    deflated = Zopfli.deflate fixture, format: :zlib
+
+    uncompressed = Zlib::Inflate.inflate deflated
+
+    expect(uncompressed).to eq fixture
+  end
+
+  it 'works (gzip format)' do
+    deflated = Zopfli.deflate fixture, format: :gzip
+
+    uncompressed = Zlib::GzipReader.wrap(StringIO.new(deflated), &:read)
+
+    expect(uncompressed).to eq fixture
+  end
+
+  it 'works (deflate format)' do
+    deflated = Zopfli.deflate fixture, format: :deflate
+    p fixture.bytesize, deflated.bytesize
+  end
+
+  context 'benchmark' do
+    it 'seq' do
+      data = 10.times.map { fixture.dup }
+      deflates = nil
+      t = Benchmark.realtime { deflates = data.map { |datum| Zopfli.deflate datum } }
+      deflates.each { |deflate| expect(Zlib::Inflate.inflate(deflate)).to eq fixture }
+      puts t
+      # => 4.483513999963179
+    end
+
+    it 'thread' do
+      q = Queue.new
+      10.times { q.push fixture.dup }
+      10.times { q.push nil }
+      w = 10.times.map do
+        Thread.new do
+          deflates = []
+          loop do
+            datum = q.pop
+            break if datum.nil?
+            deflates << Zopfli.deflate(datum)
+          end
+          deflates
+        end
+      end
+      deflates = nil
+      t = Benchmark.realtime { deflates = w.map(&:value) }
+      deflates.flatten.each { |deflate| expect(Zlib::Inflate.inflate(deflate)).to eq fixture }
+      puts t
+      # => 2.6099140000296757
+      # => 0.9405940000433475 (w/o gvl)
+    end
+  end
+end

data/vendor/zopfli/{blocksplitter.c → src/zopfli/blocksplitter.c}

@@ -24,7 +24,6 @@ Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala)
 #include <stdlib.h>
 
 #include "deflate.h"
-#include "lz77.h"
 #include "squeeze.h"
 #include "tree.h"
 #include "util.h"
@@ -39,9 +38,10 @@ typedef double FindMinimumFun(size_t i, void* context);
 /*
 Finds minimum of function f(i) where is is of type size_t, f(i) is of type
 double, i is in range start-end (excluding end).
+Outputs the minimum value in *smallest and returns the index of this value.
 */
 static size_t FindMinimum(FindMinimumFun f, void* context,
-                          size_t start, size_t end) {
+                          size_t start, size_t end, double* smallest) {
   if (end - start < 1024) {
     double best = ZOPFLI_LARGE_FLOAT;
     size_t result = start;
@@ -53,6 +53,7 @@ static size_t FindMinimum(FindMinimumFun f, void* context,
         result = i;
       }
     }
+    *smallest = best;
     return result;
   } else {
     /* Try to find minimum faster by recursively checking multiple points. */
@@ -88,6 +89,7 @@ static size_t FindMinimum(FindMinimumFun f, void* context,
       pos = p[besti];
       lastbest = best;
     }
+    *smallest = lastbest;
     return pos;
 #undef NUM
   }
@@ -103,16 +105,13 @@ dists: ll77 distances
 lstart: start of block
 lend: end of block (not inclusive)
 */
-static double EstimateCost(const unsigned short* litlens,
-                           const unsigned short* dists,
+static double EstimateCost(const ZopfliLZ77Store* lz77,
                            size_t lstart, size_t lend) {
-  return ZopfliCalculateBlockSize(litlens, dists, lstart, lend, 2);
+  return ZopfliCalculateBlockSizeAutoType(lz77, lstart, lend);
 }
 
 typedef struct SplitCostContext {
-  const unsigned short* litlens;
-  const unsigned short* dists;
-  size_t llsize;
+  const ZopfliLZ77Store* lz77;
   size_t start;
   size_t end;
 } SplitCostContext;
@@ -125,23 +124,20 @@ type: FindMinimumFun
 */
 static double SplitCost(size_t i, void* context) {
   SplitCostContext* c = (SplitCostContext*)context;
-  return EstimateCost(c->litlens, c->dists, c->start, i) +
-      EstimateCost(c->litlens, c->dists, i, c->end);
+  return EstimateCost(c->lz77, c->start, i) + EstimateCost(c->lz77, i, c->end);
 }
 
 static void AddSorted(size_t value, size_t** out, size_t* outsize) {
   size_t i;
   ZOPFLI_APPEND_DATA(value, out, outsize);
-  if (*outsize > 0) {
-    for (i = 0; i < *outsize - 1; i++) {
-      if ((*out)[i] > value) {
-        size_t j;
-        for (j = *outsize - 1; j > i; j--) {
-          (*out)[j] = (*out)[j - 1];
-        }
-        (*out)[i] = value;
-        break;
+  for (i = 0; i + 1 < *outsize; i++) {
+    if ((*out)[i] > value) {
+      size_t j;
+      for (j = *outsize - 1; j > i; j--) {
+        (*out)[j] = (*out)[j - 1];
       }
+      (*out)[i] = value;
+      break;
     }
   }
 }
@@ -149,9 +145,8 @@ static void AddSorted(size_t value, size_t** out, size_t* outsize) {
 /*
 Prints the block split points as decimal and hex values in the terminal.
 */
-static void PrintBlockSplitPoints(const unsigned short* litlens,
-                                  const unsigned short* dists,
-                                  size_t llsize, const size_t* lz77splitpoints,
+static void PrintBlockSplitPoints(const ZopfliLZ77Store* lz77,
+                                  const size_t* lz77splitpoints,
                                   size_t nlz77points) {
   size_t* splitpoints = 0;
   size_t npoints = 0;
@@ -160,8 +155,8 @@ static void PrintBlockSplitPoints(const unsigned short* litlens,
   index values. */
   size_t pos = 0;
   if (nlz77points > 0) {
-    for (i = 0; i < llsize; i++) {
-      size_t length = dists[i] == 0 ? 1 : litlens[i];
+    for (i = 0; i < lz77->size; i++) {
+      size_t length = lz77->dists[i] == 0 ? 1 : lz77->litlens[i];
       if (lz77splitpoints[npoints] == i) {
         ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints);
         if (npoints == nlz77points) break;
@@ -188,7 +183,7 @@ static void PrintBlockSplitPoints(const unsigned short* litlens,
 Finds next block to try to split, the largest of the available ones.
 The largest is chosen to make sure that if only a limited amount of blocks is
 requested, their sizes are spread evenly.
-llsize: the size of the LL77 data, which is the size of the done array here.
+lz77size: the size of the LL77 data, which is the size of the done array here.
 done: array indicating which blocks starting at that position are no longer
     splittable (splitting them increases rather than decreases cost).
 splitpoints: the splitpoints found so far.
@@ -198,7 +193,7 @@ lend: output variable, giving end of block.
 returns 1 if a block was found, 0 if no block found (all are done).
 */
 static int FindLargestSplittableBlock(
-    size_t llsize, const unsigned char* done,
+    size_t lz77size, const unsigned char* done,
     const size_t* splitpoints, size_t npoints,
     size_t* lstart, size_t* lend) {
   size_t longest = 0;
@@ -206,7 +201,7 @@ static int FindLargestSplittableBlock(
   size_t i;
   for (i = 0; i <= npoints; i++) {
     size_t start = i == 0 ? 0 : splitpoints[i - 1];
-    size_t end = i == npoints ? llsize - 1 : splitpoints[i];
+    size_t end = i == npoints ? lz77size - 1 : splitpoints[i];
     if (!done[start] && end - start > longest) {
       *lstart = start;
       *lend = end;
@@ -218,9 +213,7 @@ static int FindLargestSplittableBlock(
 }
 
 void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
-                          const unsigned short* litlens,
-                          const unsigned short* dists,
-                          size_t llsize, size_t maxblocks,
+                          const ZopfliLZ77Store* lz77, size_t maxblocks,
                           size_t** splitpoints, size_t* npoints) {
   size_t lstart, lend;
   size_t i;
@@ -229,14 +222,14 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
   unsigned char* done;
   double splitcost, origcost;
 
-  if (llsize < 10) return;  /* This code fails on tiny files. */
+  if (lz77->size < 10) return;  /* This code fails on tiny files. */
 
-  done = (unsigned char*)malloc(llsize);
+  done = (unsigned char*)malloc(lz77->size);
   if (!done) exit(-1); /* Allocation failed. */
-  for (i = 0; i < llsize; i++) done[i] = 0;
+  for (i = 0; i < lz77->size; i++) done[i] = 0;
 
   lstart = 0;
-  lend = llsize;
+  lend = lz77->size;
   for (;;) {
     SplitCostContext c;
 
@@ -244,20 +237,16 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
       break;
     }
 
-    c.litlens = litlens;
-    c.dists = dists;
-    c.llsize = llsize;
+    c.lz77 = lz77;
     c.start = lstart;
     c.end = lend;
     assert(lstart < lend);
-    llpos = FindMinimum(SplitCost, &c, lstart + 1, lend);
+    llpos = FindMinimum(SplitCost, &c, lstart + 1, lend, &splitcost);
 
     assert(llpos > lstart);
     assert(llpos < lend);
 
-    splitcost = EstimateCost(litlens, dists, lstart, llpos) +
-        EstimateCost(litlens, dists, llpos, lend);
-    origcost = EstimateCost(litlens, dists, lstart, lend);
+    origcost = EstimateCost(lz77, lstart, lend);
 
     if (splitcost > origcost || llpos == lstart + 1 || llpos == lend) {
       done[lstart] = 1;
@@ -267,7 +256,7 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
     }
 
     if (!FindLargestSplittableBlock(
-        llsize, done, *splitpoints, *npoints, &lstart, &lend)) {
+        lz77->size, done, *splitpoints, *npoints, &lstart, &lend)) {
       break;  /* No further split will probably reduce compression. */
     }
 
@@ -277,7 +266,7 @@ void ZopfliBlockSplitLZ77(const ZopfliOptions* options,
   }
 
   if (options->verbose) {
-    PrintBlockSplitPoints(litlens, dists, llsize, *splitpoints, *npoints);
+    PrintBlockSplitPoints(lz77, *splitpoints, *npoints);
   }
 
   free(done);
@@ -292,25 +281,22 @@ void ZopfliBlockSplit(const ZopfliOptions* options,
   size_t* lz77splitpoints = 0;
   size_t nlz77points = 0;
   ZopfliLZ77Store store;
+  ZopfliHash hash;
+  ZopfliHash* h = &hash;
 
-  ZopfliInitLZ77Store(&store);
-
-  s.options = options;
-  s.blockstart = instart;
-  s.blockend = inend;
-#ifdef ZOPFLI_LONGEST_MATCH_CACHE
-  s.lmc = 0;
-#endif
+  ZopfliInitLZ77Store(in, &store);
+  ZopfliInitBlockState(options, instart, inend, 0, &s);
+  ZopfliAllocHash(ZOPFLI_WINDOW_SIZE, h);
 
   *npoints = 0;
   *splitpoints = 0;
 
   /* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal
   results in better blocks. */
-  ZopfliLZ77Greedy(&s, in, instart, inend, &store);
+  ZopfliLZ77Greedy(&s, in, instart, inend, &store, h);
 
   ZopfliBlockSplitLZ77(options,
-                       store.litlens, store.dists, store.size, maxblocks,
+                       &store, maxblocks,
                        &lz77splitpoints, &nlz77points);
 
   /* Convert LZ77 positions to positions in the uncompressed input. */
@@ -328,7 +314,9 @@ void ZopfliBlockSplit(const ZopfliOptions* options,
   assert(*npoints == nlz77points);
 
   free(lz77splitpoints);
+  ZopfliCleanBlockState(&s);
   ZopfliCleanLZ77Store(&store);
+  ZopfliCleanHash(h);
 }
 
 void ZopfliBlockSplitSimple(const unsigned char* in,