libdeflate 0.1.1 → 0.2.0

data/ext/libdeflate/libdeflate/Makefile.msc

@@ -1,64 +0,0 @@
-#
-# Makefile for the Microsoft toolchain
-#
-# Usage:
-#	nmake /f Makefile.msc
-#
-
-.SUFFIXES: .c .obj .dllobj
-
-CC = cl
-LD = link
-AR = lib
-CFLAGS = /MD /O2 -I. -Icommon
-LDFLAGS =
-
-STATIC_LIB = libdeflatestatic.lib
-SHARED_LIB = libdeflate.dll
-IMPORT_LIB = libdeflate.lib
-
-STATIC_LIB_OBJ =			\
-	  lib/aligned_malloc.obj	\
-	  lib/adler32.obj		\
-	  lib/crc32.obj			\
-	  lib/deflate_compress.obj	\
-	  lib/deflate_decompress.obj	\
-	  lib/gzip_compress.obj		\
-	  lib/gzip_decompress.obj	\
-	  lib/x86_cpu_features.obj	\
-	  lib/zlib_compress.obj		\
-	  lib/zlib_decompress.obj
-
-SHARED_LIB_OBJ = $(STATIC_LIB_OBJ:.obj=.dllobj)
-
-PROG_COMMON_OBJ = programs/prog_util.obj \
-		  programs/tgetopt.obj \
-		  $(STATIC_LIB)
-
-PROG_CFLAGS = $(CFLAGS) -Iprograms
-
-all: $(STATIC_LIB) $(SHARED_LIB) $(IMPORT_LIB) gzip.exe gunzip.exe
-
-.c.obj:
-	$(CC) -c /Fo$@ $(CFLAGS) $**
-
-.c.dllobj:
-	$(CC) -c /Fo$@ $(CFLAGS) /DLIBDEFLATE_DLL $**
-
-$(STATIC_LIB): $(STATIC_LIB_OBJ)
-	$(AR) $(ARFLAGS) -out:$@ $(STATIC_LIB_OBJ)
-
-$(SHARED_LIB): $(SHARED_LIB_OBJ)
-	$(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPORT_LIB) $(SHARED_LIB_OBJ)
-
-$(IMPORT_LIB): $(SHARED_LIB)
-
-gzip.exe:programs/gzip.obj $(PROG_COMMON_OBJ)
-	$(LD) $(LDFLAGS) -out:$@ $**
-
-gunzip.exe:gzip.exe
-	copy $** $@
-
-clean:
-	-del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \
-		lib\*.obj lib\*.dllobj programs\*.obj 2>nul

data/ext/libdeflate/libdeflate/NEWS

@@ -1,57 +0,0 @@
-Version 0.7:
-	Fixed a very rare bug that caused data to be compressed incorrectly.
-	The bug affected compression levels 7 and below since libdeflate v0.2.
-	Although there have been no user reports of the bug, and I believe it
-	would have been highly unlikely to encounter on realistic data, it could
-	occur on data specially crafted to reproduce it.
-
-	Fixed a compilation error when building with clang 3.7.
-
-Version 0.6:
-	Various improvements to the gzip program's behavior.
-
-	Faster CRC-32 on AVX-capable processors.
-
-	Other minor changes.
-
-Version 0.5:
-	The CRC-32 checksum algorithm has been optimized with carryless
-	multiplication instructions for x86_64 (PCLMUL).  This speeds up gzip
-	compression and decompression.
-
-	Build fixes for certain platforms and compilers.
-
-	Added more test programs and scripts.
-
-	libdeflate is now entirely MIT-licensed.
-
-Version 0.4:
-	The Adler-32 checksum algorithm has been optimized with vector
-	instructions for x86_64 (SSE2 and AVX2) and ARM (NEON).  This speeds up
-	zlib compression and decompression.
-
-	To avoid naming collisions, functions and definitions in libdeflate's
-	API have been renamed to be prefixed with "libdeflate_" or
-	"LIBDEFLATE_".  Programs using the old API will need to be updated.
-
-	Various bug fixes and other improvements.
-
-Version 0.3:
-	Some bug fixes and other minor changes.
-
-Version 0.2:
-	Implemented a new block splitting algorithm which typically improves the
-	compression ratio slightly at all compression levels.
-
-	The compressor now outputs each block using the cheapest type (dynamic
-	Huffman, static Huffman, or uncompressed).
-
-	The gzip program has received an overhaul and now behaves more like the
-	standard version.
-
-	Build system updates, including: some build options were changed and
-	some build options were removed, and the default 'make' target now
-	includes the gzip program as well as the library.
-
-Version 0.1:
-	Initial official release.

data/ext/libdeflate/libdeflate/README.md

@@ -1,170 +0,0 @@
-# Overview
-
-libdeflate is a library for fast, whole-buffer DEFLATE-based compression and
-decompression.
-
-The supported formats are:
-
-- DEFLATE (raw)
-- zlib (a.k.a. DEFLATE with a zlib wrapper)
-- gzip (a.k.a. DEFLATE with a gzip wrapper)
-
-libdeflate is heavily optimized.  It is significantly faster than the zlib
-library, both for compression and decompression, and especially on x86
-processors.  In addition, libdeflate provides optional high compression modes
-that provide a better compression ratio than the zlib's "level 9".
-
-libdeflate itself is a library, but the following command-line programs which
-use this library are also provided:
-
-* gzip (or gunzip), a program which mostly behaves like the standard equivalent,
-  except that it does not yet have good streaming support and therefore does not
-  yet support very large files
-* benchmark, a program for benchmarking in-memory compression and decompression
-
-# Building
-
-## For UNIX
-
-Just run `make`.  You need GNU Make and either GCC or Clang.  GCC is recommended
-because it builds slightly faster binaries.  There is no `make install` yet;
-just copy the file(s) to where you want.
-
-By default, all targets are built, including the library and programs, with the
-exception of the `benchmark` program.  `make help` shows the available targets.
-There are also several options which can be set on the `make` command line.  See
-the Makefile for details.
-
-## For Windows
-
-MinGW (GCC) is the recommended compiler to use when building binaries for
-Windows.  MinGW can be used on either Windows or Linux.  On Windows, you'll need
-the compiler as well as GNU Make and basic UNIX tools such as `sh`.  This is
-most easily set up with Cygwin, but some standalone MinGW distributions for
-Windows also work.  Or, on Linux, you'll need to install the `mingw-w64-gcc` or
-similarly-named package.  Once ready, do the build using a command like:
-
-    $ make CC=x86_64-w64-mingw32-gcc
-
-Some MinGW distributions for Windows may require `CC=gcc` instead.
-
-Windows binaries prebuilt with MinGW may also be downloaded from
-https://github.com/ebiggers/libdeflate/releases.
-
-Alternatively, a separate Makefile, `Makefile.msc`, is provided for the tools
-that come with Visual Studio, for those who strongly prefer that toolchain.
-
-As usual, 64-bit binaries are faster than 32-bit binaries and should be
-preferred whenever possible.
-
-# API
-
-libdeflate has a simple API that is not zlib-compatible.  You can create
-compressors and decompressors and use them to compress or decompress buffers.
-See libdeflate.h for details.
-
-There is currently no support for streaming.  This has been considered, but it
-always significantly increases complexity and slows down fast paths.
-Unfortunately, at this point it remains a future TODO.  So: if your application
-compresses data in "chunks", say, less than 1 MB in size, then libdeflate is a
-great choice for you; that's what it's designed to do.  This is perfect for
-certain use cases such as transparent filesystem compression.  But if your
-application compresses large files as a single compressed stream, similarly to
-the `gzip` program, then libdeflate isn't for you.
-
-Note that with chunk-based compression, you generally should have the
-uncompressed size of each chunk stored outside of the compressed data itself.
-This enables you to allocate an output buffer of the correct size without
-guessing.  However, libdeflate's decompression routines do optionally provide
-the actual number of output bytes in case you need it.
-
-# DEFLATE vs. zlib vs. gzip
-
-The DEFLATE format ([rfc1951](https://www.ietf.org/rfc/rfc1951.txt)), the zlib
-format ([rfc1950](https://www.ietf.org/rfc/rfc1950.txt)), and the gzip format
-([rfc1952](https://www.ietf.org/rfc/rfc1952.txt)) are commonly confused with
-each other as well as with the [zlib software library](http://zlib.net), which
-actually supports all three formats.  libdeflate (this library) also supports
-all three formats.
-
-Briefly, DEFLATE is a raw compressed stream, whereas zlib and gzip are different
-wrappers for this stream.  Both zlib and gzip include checksums, but gzip can
-include extra information such as the original filename.  Generally, you should
-choose a format as follows:
-
-- If you are compressing whole files with no subdivisions, similar to the `gzip`
-  program, you probably should use the gzip format.
-- Otherwise, if you don't need the features of the gzip header and footer but do
-  still want a checksum for corruption detection, you probably should use the
-  zlib format.
-- Otherwise, you probably should use raw DEFLATE.  This is ideal if you don't
-  need checksums, e.g. because they're simply not needed for your use case or
-  because you already compute your own checksums that are stored separately from
-  the compressed stream.
-
-Note that gzip and zlib streams can be distinguished from each other based on
-their starting bytes, but this is not necessarily true of raw DEFLATE streams.
-
-# Compression levels
-
-An often-underappreciated fact of compression formats such as DEFLATE is that
-there are an enormous number of different ways that a given input could be
-compressed.  Different algorithms and different amounts of computation time will
-result in different compression ratios, while remaining equally compatible with
-the decompressor.
-
-For this reason, the commonly used zlib library provides nine compression
-levels.  Level 1 is the fastest but provides the worst compression; level 9
-provides the best compression but is the slowest.  It defaults to level 6.
-libdeflate uses this same design but is designed to improve on both zlib's
-performance *and* compression ratio at every compression level.  In addition,
-libdeflate's levels go [up to 12](https://xkcd.com/670/) to make room for a
-minimum-cost-path based algorithm (sometimes called "optimal parsing") that can
-significantly improve on zlib's compression ratio.
-
-If you are using DEFLATE (or zlib, or gzip) in your application, you should test
-different levels to see which works best for your application.
-
-# Motivation
-
-Despite DEFLATE's widespread use mainly through the zlib library, in the
-compression community this format from the early 1990s is often considered
-obsolete.  And in a few significant ways, it is.
-
-So why implement DEFLATE at all, instead of focusing entirely on
-bzip2/LZMA/xz/LZ4/LZX/ZSTD/Brotli/LZHAM/LZFSE/[insert cool new format here]?
-
-To do something better, you need to understand what came before.  And it turns
-out that most ideas from DEFLATE are still relevant.  Many of the newer formats
-share a similar structure as DEFLATE, with different tweaks.  The effects of
-trivial but very useful tweaks, such as increasing the sliding window size, are
-often confused with the effects of nontrivial but less useful tweaks.  And
-actually, many of these formats are similar enough that common algorithms and
-optimizations (e.g. those dealing with LZ77 matchfinding) can be reused.
-
-In addition, comparing compressors fairly is difficult because the performance
-of a compressor depends heavily on optimizations which are not intrinsic to the
-compression format itself.  In this respect, the zlib library sometimes compares
-poorly to certain newer code because zlib is not well optimized for modern
-processors.  libdeflate addresses this by providing an optimized DEFLATE
-implementation which can be used for benchmarking purposes.  And, of course,
-real applications can use it as well.
-
-That being said, I have also started [a separate
-project](https://github.com/ebiggers/xpack) for an experimental, more modern
-compression format.
-
-# License
-
-libdeflate is [MIT-licensed](COPYING).
-
-Additional notes (informational only):
-
-- I am not aware of any patents covering libdeflate.
-
-- Old versions of libdeflate were public domain; I only started copyrighting
-  changes in newer versions.  Portions of the source code that have not been
-  changed since being released in a public domain version can theoretically
-  still be used as public domain if you want to.  But for practical purposes, it
-  probably would be easier to just take the MIT license option, which is nearly
-  the same anyway.

data/ext/libdeflate/libdeflate/common/common_defs.h

@@ -1,351 +0,0 @@
-/*
- * common_defs.h
- *
- * Copyright 2016 Eric Biggers
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef COMMON_COMMON_DEFS_H
-#define COMMON_COMMON_DEFS_H
-
-#ifdef __GNUC__
-#  include "compiler_gcc.h"
-#elif defined(_MSC_VER)
-#  include "compiler_msc.h"
-#else
-#  pragma message("Unrecognized compiler.  Please add a header file for your compiler.  Compilation will proceed, but performance may suffer!")
-#endif
-
-/* ========================================================================== */
-/*                              Type definitions                              */
-/* ========================================================================== */
-
-#include <stddef.h> /* size_t */
-
-#ifndef __bool_true_false_are_defined
-#  include <stdbool.h> /* bool */
-#endif
-
-/* Fixed-width integer types */
-#ifndef PRIu32
-#  include <inttypes.h>
-#endif
-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-typedef int8_t s8;
-typedef int16_t s16;
-typedef int32_t s32;
-typedef int64_t s64;
-
-/*
- * Word type of the target architecture.  Use 'size_t' instead of 'unsigned
- * long' to account for platforms such as Windows that use 32-bit 'unsigned
- * long' on 64-bit architectures.
- */
-typedef size_t machine_word_t;
-
-/* Number of bytes in a word */
-#define WORDBYTES	((int)sizeof(machine_word_t))
-
-/* Number of bits in a word */
-#define WORDBITS	(8 * WORDBYTES)
-
-/* ========================================================================== */
-/*                         Optional compiler features                         */
-/* ========================================================================== */
-
-/* LIBEXPORT - export a function from a shared library */
-#ifndef LIBEXPORT
-#  define LIBEXPORT
-#endif
-
-/* inline - suggest that a function be inlined */
-#ifndef inline
-#  define inline
-#endif
-
-/* forceinline - force a function to be inlined, if possible */
-#ifndef forceinline
-#  define forceinline inline
-#endif
-
-/* restrict - annotate a non-aliased pointer */
-#ifndef restrict
-#  define restrict
-#endif
-
-/* likely(expr) - hint that an expression is usually true */
-#ifndef likely
-#  define likely(expr)		(expr)
-#endif
-
-/* unlikely(expr) - hint that an expression is usually false */
-#ifndef unlikely
-#  define unlikely(expr)	(expr)
-#endif
-
-/* prefetchr(addr) - prefetch into L1 cache for read */
-#ifndef prefetchr
-#  define prefetchr(addr)
-#endif
-
-/* prefetchw(addr) - prefetch into L1 cache for write */
-#ifndef prefetchw
-#  define prefetchw(addr)
-#endif
-
-/* Does the compiler support the 'target' function attribute? */
-#ifndef COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
-#  define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0
-#endif
-
-/* Are target-specific intrinsics supported in 'target' attribute functions? */
-#ifndef COMPILER_SUPPORTS_TARGET_INTRINSICS
-#  define COMPILER_SUPPORTS_TARGET_INTRINSICS 0
-#endif
-
-/* Which targets are supported with the 'target' function attribute? */
-#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET
-#  define COMPILER_SUPPORTS_PCLMUL_TARGET 0
-#endif
-#ifndef COMPILER_SUPPORTS_BMI2_TARGET
-#  define COMPILER_SUPPORTS_BMI2_TARGET 0
-#endif
-#ifndef COMPILER_SUPPORTS_AVX_TARGET
-#  define COMPILER_SUPPORTS_AVX_TARGET 0
-#endif
-#ifndef COMPILER_SUPPORTS_AVX2_TARGET
-#  define COMPILER_SUPPORTS_AVX2_TARGET 0
-#endif
-
-/* _aligned_attribute(n) - declare that the annotated variable, or variables of
- * the annotated type, are to be aligned on n-byte boundaries */
-#ifndef _aligned_attribute
-#endif
-
-/* ========================================================================== */
-/*                          Miscellaneous macros                              */
-/* ========================================================================== */
-
-#define ARRAY_LEN(A)		(sizeof(A) / sizeof((A)[0]))
-#define MIN(a, b)		((a) <= (b) ? (a) : (b))
-#define MAX(a, b)		((a) >= (b) ? (a) : (b))
-#define DIV_ROUND_UP(n, d)	(((n) + (d) - 1) / (d))
-#define STATIC_ASSERT(expr)	((void)sizeof(char[1 - 2 * !(expr)]))
-#define ALIGN(n, a)		(((n) + (a) - 1) & ~((a) - 1))
-
-/* ========================================================================== */
-/*                           Endianness handling                              */
-/* ========================================================================== */
-
-/*
- * CPU_IS_LITTLE_ENDIAN() - a macro which evaluates to 1 if the CPU is little
- * endian or 0 if it is big endian.  The macro should be defined in a way such
- * that the compiler can evaluate it at compilation time.  If not defined, a
- * fallback is used.
- */
-#ifndef CPU_IS_LITTLE_ENDIAN
-static forceinline int CPU_IS_LITTLE_ENDIAN(void)
-{
-	union {
-		unsigned int v;
-		unsigned char b;
-	} u;
-	u.v = 1;
-	return u.b;
-}
-#endif
-
-/* bswap16(n) - swap the bytes of a 16-bit integer */
-#ifndef bswap16
-static forceinline u16 bswap16(u16 n)
-{
-	return (n << 8) | (n >> 8);
-}
-#endif
-
-/* bswap32(n) - swap the bytes of a 32-bit integer */
-#ifndef bswap32
-static forceinline u32 bswap32(u32 n)
-{
-	return ((n & 0x000000FF) << 24) |
-	       ((n & 0x0000FF00) << 8) |
-	       ((n & 0x00FF0000) >> 8) |
-	       ((n & 0xFF000000) >> 24);
-}
-#endif
-
-/* bswap64(n) - swap the bytes of a 64-bit integer */
-#ifndef bswap64
-static forceinline u64 bswap64(u64 n)
-{
-	return ((n & 0x00000000000000FF) << 56) |
-	       ((n & 0x000000000000FF00) << 40) |
-	       ((n & 0x0000000000FF0000) << 24) |
-	       ((n & 0x00000000FF000000) << 8) |
-	       ((n & 0x000000FF00000000) >> 8) |
-	       ((n & 0x0000FF0000000000) >> 24) |
-	       ((n & 0x00FF000000000000) >> 40) |
-	       ((n & 0xFF00000000000000) >> 56);
-}
-#endif
-
-#define le16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap16(n))
-#define le32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap32(n))
-#define le64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap64(n))
-#define be16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap16(n) : (n))
-#define be32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap32(n) : (n))
-#define be64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap64(n) : (n))
-
-/* ========================================================================== */
-/*                          Unaligned memory accesses                         */
-/* ========================================================================== */
-
-/*
- * UNALIGNED_ACCESS_IS_FAST should be defined to 1 if unaligned memory accesses
- * can be performed efficiently on the target platform.
- */
-#ifndef UNALIGNED_ACCESS_IS_FAST
-#  define UNALIGNED_ACCESS_IS_FAST 0
-#endif
-
-/*
- * DEFINE_UNALIGNED_TYPE(type) - a macro that, given an integer type 'type',
- * defines load_type_unaligned(addr) and store_type_unaligned(v, addr) functions
- * which load and store variables of type 'type' from/to unaligned memory
- * addresses.  If not defined, a fallback is used.
- */
-#ifndef DEFINE_UNALIGNED_TYPE
-
-/*
- * Although memcpy() may seem inefficient, it *usually* gets optimized
- * appropriately by modern compilers.  It's portable and may be the best we can
- * do for a fallback...
- */
-#include <string.h>
-
-#define DEFINE_UNALIGNED_TYPE(type)				\
-								\
-static forceinline type						\
-load_##type##_unaligned(const void *p)				\
-{								\
-	type v;							\
-	memcpy(&v, p, sizeof(v));				\
-	return v;						\
-}								\
-								\
-static forceinline void						\
-store_##type##_unaligned(type v, void *p)			\
-{								\
-	memcpy(p, &v, sizeof(v));				\
-}
-
-#endif /* !DEFINE_UNALIGNED_TYPE */
-
-/* ========================================================================== */
-/*                             Bit scan functions                             */
-/* ========================================================================== */
-
-/*
- * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
- * significant end) of the *most* significant 1 bit in the input value.  The
- * input value must be nonzero!
- */
-
-#ifndef bsr32
-static forceinline unsigned
-bsr32(u32 n)
-{
-	unsigned i = 0;
-	while ((n >>= 1) != 0)
-		i++;
-	return i;
-}
-#endif
-
-#ifndef bsr64
-static forceinline unsigned
-bsr64(u64 n)
-{
-	unsigned i = 0;
-	while ((n >>= 1) != 0)
-		i++;
-	return i;
-}
-#endif
-
-static forceinline unsigned
-bsrw(machine_word_t n)
-{
-	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-	if (WORDBITS == 32)
-		return bsr32(n);
-	else
-		return bsr64(n);
-}
-
-/*
- * Bit Scan Forward (BSF) - find the 0-based index (relative to the least
- * significant end) of the *least* significant 1 bit in the input value.  The
- * input value must be nonzero!
- */
-
-#ifndef bsf32
-static forceinline unsigned
-bsf32(u32 n)
-{
-	unsigned i = 0;
-	while ((n & 1) == 0) {
-		i++;
-		n >>= 1;
-	}
-	return i;
-}
-#endif
-
-#ifndef bsf64
-static forceinline unsigned
-bsf64(u64 n)
-{
-	unsigned i = 0;
-	while ((n & 1) == 0) {
-		i++;
-		n >>= 1;
-	}
-	return i;
-}
-#endif
-
-static forceinline unsigned
-bsfw(machine_word_t n)
-{
-	STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
-	if (WORDBITS == 32)
-		return bsf32(n);
-	else
-		return bsf64(n);
-}
-
-#endif /* COMMON_COMMON_DEFS_H */