libdeflate 0.1.0

data/ext/libdeflate/libdeflate/Makefile.msc

@@ -0,0 +1,64 @@
+#
+# Makefile for the Microsoft toolchain
+#
+# Usage:
+#	nmake /f Makefile.msc
+#
+
+.SUFFIXES: .c .obj .dllobj
+
+CC = cl
+LD = link
+AR = lib
+CFLAGS = /MD /O2 -I. -Icommon
+LDFLAGS =
+
+STATIC_LIB = libdeflatestatic.lib
+SHARED_LIB = libdeflate.dll
+IMPORT_LIB = libdeflate.lib
+
+STATIC_LIB_OBJ =			\
+	  lib/aligned_malloc.obj	\
+	  lib/adler32.obj		\
+	  lib/crc32.obj			\
+	  lib/deflate_compress.obj	\
+	  lib/deflate_decompress.obj	\
+	  lib/gzip_compress.obj		\
+	  lib/gzip_decompress.obj	\
+	  lib/x86_cpu_features.obj	\
+	  lib/zlib_compress.obj		\
+	  lib/zlib_decompress.obj
+
+SHARED_LIB_OBJ = $(STATIC_LIB_OBJ:.obj=.dllobj)
+
+PROG_COMMON_OBJ = programs/prog_util.obj \
+		  programs/tgetopt.obj \
+		  $(STATIC_LIB)
+
+PROG_CFLAGS = $(CFLAGS) -Iprograms
+
+all: $(STATIC_LIB) $(SHARED_LIB) $(IMPORT_LIB) gzip.exe gunzip.exe
+
+.c.obj:
+	$(CC) -c /Fo$@ $(CFLAGS) $**
+
+.c.dllobj:
+	$(CC) -c /Fo$@ $(CFLAGS) /DLIBDEFLATE_DLL $**
+
+$(STATIC_LIB): $(STATIC_LIB_OBJ)
+	$(AR) $(ARFLAGS) -out:$@ $(STATIC_LIB_OBJ)
+
+$(SHARED_LIB): $(SHARED_LIB_OBJ)
+	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPORT_LIB) $(SHARED_LIB_OBJ)
+
+$(IMPORT_LIB): $(SHARED_LIB)
+
+gzip.exe:programs/gzip.obj $(PROG_COMMON_OBJ)
+	$(LD) $(LDFLAGS) -out:$@ $**
+
+gunzip.exe:gzip.exe
+	copy $** $@
+
+clean:
+	-del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \
+		lib\*.obj lib\*.dllobj programs\*.obj 2>nul

data/ext/libdeflate/libdeflate/NEWS

@@ -0,0 +1,57 @@
+Version 0.7:
+	Fixed a very rare bug that caused data to be compressed incorrectly.
+	The bug affected compression levels 7 and below since libdeflate v0.2.
+	Although there have been no user reports of the bug, and I believe it
+	would have been highly unlikely to encounter on realistic data, it could
+	occur on data specially crafted to reproduce it.
+
+	Fixed a compilation error when building with clang 3.7.
+
+Version 0.6:
+	Various improvements to the gzip program's behavior.
+
+	Faster CRC-32 on AVX-capable processors.
+
+	Other minor changes.
+
+Version 0.5:
+	The CRC-32 checksum algorithm has been optimized with carryless
+	multiplication instructions for x86_64 (PCLMUL).  This speeds up gzip
+	compression and decompression.
+
+	Build fixes for certain platforms and compilers.
+
+	Added more test programs and scripts.
+
+	libdeflate is now entirely MIT-licensed.
+
+Version 0.4:
+	The Adler-32 checksum algorithm has been optimized with vector
+	instructions for x86_64 (SSE2 and AVX2) and ARM (NEON).  This speeds up
+	zlib compression and decompression.
+
+	To avoid naming collisions, functions and definitions in libdeflate's
+	API have been renamed to be prefixed with "libdeflate_" or
+	"LIBDEFLATE_".  Programs using the old API will need to be updated.
+
+	Various bug fixes and other improvements.
+
+Version 0.3:
+	Some bug fixes and other minor changes.
+
+Version 0.2:
+	Implemented a new block splitting algorithm which typically improves the
+	compression ratio slightly at all compression levels.
+
+	The compressor now outputs each block using the cheapest type (dynamic
+	Huffman, static Huffman, or uncompressed).
+
+	The gzip program has received an overhaul and now behaves more like the
+	standard version.
+
+	Build system updates, including: some build options were changed and
+	some build options were removed, and the default 'make' target now
+	includes the gzip program as well as the library.
+
+Version 0.1:
+	Initial official release.

data/ext/libdeflate/libdeflate/README.md

@@ -0,0 +1,170 @@
+# Overview
+
+libdeflate is a library for fast, whole-buffer DEFLATE-based compression and
+decompression.
+
+The supported formats are:
+
+- DEFLATE (raw)
+- zlib (a.k.a. DEFLATE with a zlib wrapper)
+- gzip (a.k.a. DEFLATE with a gzip wrapper)
+
+libdeflate is heavily optimized.  It is significantly faster than the zlib
+library, both for compression and decompression, and especially on x86
+processors.  In addition, libdeflate provides optional high compression modes
+that provide a better compression ratio than the zlib's "level 9".
+
+libdeflate itself is a library, but the following command-line programs which
+use this library are also provided:
+
+* gzip (or gunzip), a program which mostly behaves like the standard equivalent,
+  except that it does not yet have good streaming support and therefore does not
+  yet support very large files
+* benchmark, a program for benchmarking in-memory compression and decompression
+
+# Building
+
+## For UNIX
+
+Just run `make`.  You need GNU Make and either GCC or Clang.  GCC is recommended
+because it builds slightly faster binaries.  There is no `make install` yet;
+just copy the file(s) to where you want.
+
+By default, all targets are built, including the library and programs, with the
+exception of the `benchmark` program.  `make help` shows the available targets.
+There are also several options which can be set on the `make` command line.  See
+the Makefile for details.
+
+## For Windows
+
+MinGW (GCC) is the recommended compiler to use when building binaries for
+Windows.  MinGW can be used on either Windows or Linux.  On Windows, you'll need
+the compiler as well as GNU Make and basic UNIX tools such as `sh`.  This is
+most easily set up with Cygwin, but some standalone MinGW distributions for
+Windows also work.  Or, on Linux, you'll need to install the `mingw-w64-gcc` or
+similarly-named package.  Once ready, do the build using a command like:
+
+    $ make CC=x86_64-w64-mingw32-gcc
+
+Some MinGW distributions for Windows may require `CC=gcc` instead.
+
+Windows binaries prebuilt with MinGW may also be downloaded from
+https://github.com/ebiggers/libdeflate/releases.
+
+Alternatively, a separate Makefile, `Makefile.msc`, is provided for the tools
+that come with Visual Studio, for those who strongly prefer that toolchain.
+
+As usual, 64-bit binaries are faster than 32-bit binaries and should be
+preferred whenever possible.
+
+# API
+
+libdeflate has a simple API that is not zlib-compatible.  You can create
+compressors and decompressors and use them to compress or decompress buffers.
+See libdeflate.h for details.
+
+There is currently no support for streaming.  This has been considered, but it
+always significantly increases complexity and slows down fast paths.
+Unfortunately, at this point it remains a future TODO.  So: if your application
+compresses data in "chunks", say, less than 1 MB in size, then libdeflate is a
+great choice for you; that's what it's designed to do.  This is perfect for
+certain use cases such as transparent filesystem compression.  But if your
+application compresses large files as a single compressed stream, similarly to
+the `gzip` program, then libdeflate isn't for you.
+
+Note that with chunk-based compression, you generally should have the
+uncompressed size of each chunk stored outside of the compressed data itself.
+This enables you to allocate an output buffer of the correct size without
+guessing.  However, libdeflate's decompression routines do optionally provide
+the actual number of output bytes in case you need it.
+
+# DEFLATE vs. zlib vs. gzip
+
+The DEFLATE format ([rfc1951](https://www.ietf.org/rfc/rfc1951.txt)), the zlib
+format ([rfc1950](https://www.ietf.org/rfc/rfc1950.txt)), and the gzip format
+([rfc1952](https://www.ietf.org/rfc/rfc1952.txt)) are commonly confused with
+each other as well as with the [zlib software library](http://zlib.net), which
+actually supports all three formats.  libdeflate (this library) also supports
+all three formats.
+
+Briefly, DEFLATE is a raw compressed stream, whereas zlib and gzip are different
+wrappers for this stream.  Both zlib and gzip include checksums, but gzip can
+include extra information such as the original filename.  Generally, you should
+choose a format as follows:
+
+- If you are compressing whole files with no subdivisions, similar to the `gzip`
+  program, you probably should use the gzip format.
+- Otherwise, if you don't need the features of the gzip header and footer but do
+  still want a checksum for corruption detection, you probably should use the
+  zlib format.
+- Otherwise, you probably should use raw DEFLATE.  This is ideal if you don't
+  need checksums, e.g. because they're simply not needed for your use case or
+  because you already compute your own checksums that are stored separately from
+  the compressed stream.
+
+Note that gzip and zlib streams can be distinguished from each other based on
+their starting bytes, but this is not necessarily true of raw DEFLATE streams.
+
+# Compression levels
+
+An often-underappreciated fact of compression formats such as DEFLATE is that
+there are an enormous number of different ways that a given input could be
+compressed.  Different algorithms and different amounts of computation time will
+result in different compression ratios, while remaining equally compatible with
+the decompressor.
+
+For this reason, the commonly used zlib library provides nine compression
+levels.  Level 1 is the fastest but provides the worst compression; level 9
+provides the best compression but is the slowest.  It defaults to level 6.
+libdeflate uses this same design but is designed to improve on both zlib's
+performance *and* compression ratio at every compression level.  In addition,
+libdeflate's levels go [up to 12](https://xkcd.com/670/) to make room for a
+minimum-cost-path based algorithm (sometimes called "optimal parsing") that can
+significantly improve on zlib's compression ratio.
+
+If you are using DEFLATE (or zlib, or gzip) in your application, you should test
+different levels to see which works best for your application.
+
+# Motivation
+
+Despite DEFLATE's widespread use mainly through the zlib library, in the
+compression community this format from the early 1990s is often considered
+obsolete.  And in a few significant ways, it is.
+
+So why implement DEFLATE at all, instead of focusing entirely on
+bzip2/LZMA/xz/LZ4/LZX/ZSTD/Brotli/LZHAM/LZFSE/[insert cool new format here]?
+
+To do something better, you need to understand what came before.  And it turns
+out that most ideas from DEFLATE are still relevant.  Many of the newer formats
+share a similar structure as DEFLATE, with different tweaks.  The effects of
+trivial but very useful tweaks, such as increasing the sliding window size, are
+often confused with the effects of nontrivial but less useful tweaks.  And
+actually, many of these formats are similar enough that common algorithms and
+optimizations (e.g. those dealing with LZ77 matchfinding) can be reused.
+
+In addition, comparing compressors fairly is difficult because the performance
+of a compressor depends heavily on optimizations which are not intrinsic to the
+compression format itself.  In this respect, the zlib library sometimes compares
+poorly to certain newer code because zlib is not well optimized for modern
+processors.  libdeflate addresses this by providing an optimized DEFLATE
+implementation which can be used for benchmarking purposes.  And, of course,
+real applications can use it as well.
+
+That being said, I have also started [a separate
+project](https://github.com/ebiggers/xpack) for an experimental, more modern
+compression format.
+
+# License
+
+libdeflate is [MIT-licensed](COPYING).
+
+Additional notes (informational only):
+
+- I am not aware of any patents covering libdeflate.
+
+- Old versions of libdeflate were public domain; I only started copyrighting
+  changes in newer versions.  Portions of the source code that have not been
+  changed since being released in a public domain version can theoretically
+  still be used as public domain if you want to.  But for practical purposes, it
+  probably would be easier to just take the MIT license option, which is nearly
+  the same anyway.

data/ext/libdeflate/libdeflate/common/common_defs.h

@@ -0,0 +1,351 @@
+/*
+ * common_defs.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef COMMON_COMMON_DEFS_H
+#define COMMON_COMMON_DEFS_H
+
+#ifdef __GNUC__
+#  include "compiler_gcc.h"
+#elif defined(_MSC_VER)
+#  include "compiler_msc.h"
+#else
+#  pragma message("Unrecognized compiler.  Please add a header file for your compiler.  Compilation will proceed, but performance may suffer!")
+#endif
+
+/* ========================================================================== */
+/*                              Type definitions                              */
+/* ========================================================================== */
+
+#include <stddef.h> /* size_t */
+
+#ifndef __bool_true_false_are_defined
+#  include <stdbool.h> /* bool */
+#endif
+
+/* Fixed-width integer types */
+#ifndef PRIu32
+#  include <inttypes.h>
+#endif
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+typedef int64_t s64;
+
+/*
+ * Word type of the target architecture.  Use 'size_t' instead of 'unsigned
+ * long' to account for platforms such as Windows that use 32-bit 'unsigned
+ * long' on 64-bit architectures.
+ */
+typedef size_t machine_word_t;
+
+/* Number of bytes in a word */
+#define WORDBYTES	((int)sizeof(machine_word_t))
+
+/* Number of bits in a word */
+#define WORDBITS	(8 * WORDBYTES)
+
+/* ========================================================================== */
+/*                         Optional compiler features                         */
+/* ========================================================================== */
+
+/* LIBEXPORT - export a function from a shared library */
+#ifndef LIBEXPORT
+#  define LIBEXPORT
+#endif
+
+/* inline - suggest that a function be inlined */
+#ifndef inline
+#  define inline
+#endif
+
+/* forceinline - force a function to be inlined, if possible */
+#ifndef forceinline
+#  define forceinline inline
+#endif
+
+/* restrict - annotate a non-aliased pointer */
+#ifndef restrict
+#  define restrict
+#endif
+
+/* likely(expr) - hint that an expression is usually true */
+#ifndef likely
+#  define likely(expr)		(expr)
+#endif
+
+/* unlikely(expr) - hint that an expression is usually false */
+#ifndef unlikely
+#  define unlikely(expr)	(expr)
+#endif
+
+/* prefetchr(addr) - prefetch into L1 cache for read */
+#ifndef prefetchr
+#  define prefetchr(addr)
+#endif
+
+/* prefetchw(addr) - prefetch into L1 cache for write */
+#ifndef prefetchw
+#  define prefetchw(addr)
+#endif
+
+/* Does the compiler support the 'target' function attribute? */
+#ifndef COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+#  define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0
+#endif
+
+/* Are target-specific intrinsics supported in 'target' attribute functions? */
+#ifndef COMPILER_SUPPORTS_TARGET_INTRINSICS
+#  define COMPILER_SUPPORTS_TARGET_INTRINSICS 0
+#endif
+
+/* Which targets are supported with the 'target' function attribute? */
+#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET
+#  define COMPILER_SUPPORTS_PCLMUL_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_BMI2_TARGET
+#  define COMPILER_SUPPORTS_BMI2_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX_TARGET
+#  define COMPILER_SUPPORTS_AVX_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX2_TARGET
+#  define COMPILER_SUPPORTS_AVX2_TARGET 0
+#endif
+
+/* _aligned_attribute(n) - declare that the annotated variable, or variables of
+ * the annotated type, are to be aligned on n-byte boundaries */
+#ifndef _aligned_attribute
+#endif
+
+/* ========================================================================== */
+/*                          Miscellaneous macros                              */
+/* ========================================================================== */
+
+#define ARRAY_LEN(A)		(sizeof(A) / sizeof((A)[0]))
+#define MIN(a, b)		((a) <= (b) ? (a) : (b))
+#define MAX(a, b)		((a) >= (b) ? (a) : (b))
+#define DIV_ROUND_UP(n, d)	(((n) + (d) - 1) / (d))
+#define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
+#define ALIGN(n, a)		(((n) + (a) - 1) & ~((a) - 1))
+
+/* ========================================================================== */
+/*                           Endianness handling                              */
+/* ========================================================================== */
+
+/*
+ * CPU_IS_LITTLE_ENDIAN() - a macro which evaluates to 1 if the CPU is little
+ * endian or 0 if it is big endian.  The macro should be defined in a way such
+ * that the compiler can evaluate it at compilation time.  If not defined, a
+ * fallback is used.
+ */
+#ifndef CPU_IS_LITTLE_ENDIAN
+static forceinline int CPU_IS_LITTLE_ENDIAN(void)
+{
+	union {
+		unsigned int v;
+		unsigned char b;
+	} u;
+	u.v = 1;
+	return u.b;
+}
+#endif
+
+/* bswap16(n) - swap the bytes of a 16-bit integer */
+#ifndef bswap16
+static forceinline u16 bswap16(u16 n)
+{
+	return (n << 8) | (n >> 8);
+}
+#endif
+
+/* bswap32(n) - swap the bytes of a 32-bit integer */
+#ifndef bswap32
+static forceinline u32 bswap32(u32 n)
+{
+	return ((n & 0x000000FF) << 24) |
+	       ((n & 0x0000FF00) << 8) |
+	       ((n & 0x00FF0000) >> 8) |
+	       ((n & 0xFF000000) >> 24);
+}
+#endif
+
+/* bswap64(n) - swap the bytes of a 64-bit integer */
+#ifndef bswap64
+static forceinline u64 bswap64(u64 n)
+{
+	return ((n & 0x00000000000000FF) << 56) |
+	       ((n & 0x000000000000FF00) << 40) |
+	       ((n & 0x0000000000FF0000) << 24) |
+	       ((n & 0x00000000FF000000) << 8) |
+	       ((n & 0x000000FF00000000) >> 8) |
+	       ((n & 0x0000FF0000000000) >> 24) |
+	       ((n & 0x00FF000000000000) >> 40) |
+	       ((n & 0xFF00000000000000) >> 56);
+}
+#endif
+
+#define le16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap16(n))
+#define le32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap32(n))
+#define le64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap64(n))
+#define be16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap16(n) : (n))
+#define be32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap32(n) : (n))
+#define be64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap64(n) : (n))
+
+/* ========================================================================== */
+/*                          Unaligned memory accesses                         */
+/* ========================================================================== */
+
+/*
+ * UNALIGNED_ACCESS_IS_FAST should be defined to 1 if unaligned memory accesses
+ * can be performed efficiently on the target platform.
+ */
+#ifndef UNALIGNED_ACCESS_IS_FAST
+#  define UNALIGNED_ACCESS_IS_FAST 0
+#endif
+
+/*
+ * DEFINE_UNALIGNED_TYPE(type) - a macro that, given an integer type 'type',
+ * defines load_type_unaligned(addr) and store_type_unaligned(v, addr) functions
+ * which load and store variables of type 'type' from/to unaligned memory
+ * addresses.  If not defined, a fallback is used.
+ */
+#ifndef DEFINE_UNALIGNED_TYPE
+
+/*
+ * Although memcpy() may seem inefficient, it *usually* gets optimized
+ * appropriately by modern compilers.  It's portable and may be the best we can
+ * do for a fallback...
+ */
+#include <string.h>
+
+#define DEFINE_UNALIGNED_TYPE(type)				\
+								\
+static forceinline type						\
+load_##type##_unaligned(const void *p)				\
+{								\
+	type v;							\
+	memcpy(&v, p, sizeof(v));				\
+	return v;						\
+}								\
+								\
+static forceinline void						\
+store_##type##_unaligned(type v, void *p)			\
+{								\
+	memcpy(p, &v, sizeof(v));				\
+}
+
+#endif /* !DEFINE_UNALIGNED_TYPE */
+
+/* ========================================================================== */
+/*                             Bit scan functions                             */
+/* ========================================================================== */
+
+/*
+ * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
+ * significant end) of the *most* significant 1 bit in the input value.  The
+ * input value must be nonzero!
+ */
+
+#ifndef bsr32
+static forceinline unsigned
+bsr32(u32 n)
+{
+	unsigned i = 0;
+	while ((n >>= 1) != 0)
+		i++;
+	return i;
+}
+#endif
+
+#ifndef bsr64
+static forceinline unsigned
+bsr64(u64 n)
+{
+	unsigned i = 0;
+	while ((n >>= 1) != 0)
+		i++;
+	return i;
+}
+#endif
+
+static forceinline unsigned
+bsrw(machine_word_t n)
+{
+	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+	if (WORDBITS == 32)
+		return bsr32(n);
+	else
+		return bsr64(n);
+}
+
+/*
+ * Bit Scan Forward (BSF) - find the 0-based index (relative to the least
+ * significant end) of the *least* significant 1 bit in the input value.  The
+ * input value must be nonzero!
+ */
+
+#ifndef bsf32
+static forceinline unsigned
+bsf32(u32 n)
+{
+	unsigned i = 0;
+	while ((n & 1) == 0) {
+		i++;
+		n >>= 1;
+	}
+	return i;
+}
+#endif
+
+#ifndef bsf64
+static forceinline unsigned
+bsf64(u64 n)
+{
+	unsigned i = 0;
+	while ((n & 1) == 0) {
+		i++;
+		n >>= 1;
+	}
+	return i;
+}
+#endif
+
+static forceinline unsigned
+bsfw(machine_word_t n)
+{
+	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+	if (WORDBITS == 32)
+		return bsf32(n);
+	else
+		return bsf64(n);
+}
+
+#endif /* COMMON_COMMON_DEFS_H */