libdeflate 0.1.0

data/ext/libdeflate/libdeflate/tools/afl-fuzz/deflate_decompress/fuzz.c

@@ -0,0 +1,28 @@
+#include <assert.h>
+#include <libdeflate.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+int main(int argc, char **argv)
+{
+	struct libdeflate_decompressor *d;
+	int ret;
+	int fd = open(argv[1], O_RDONLY);
+	struct stat stbuf;
+	assert(fd >= 0);
+	ret = fstat(fd, &stbuf);
+	assert(!ret);
+
+	char in[stbuf.st_size];
+	ret = read(fd, in, sizeof in);
+	assert(ret == sizeof in);
+
+	char out[sizeof(in) * 3];
+
+	d = libdeflate_alloc_decompressor();
+
+	libdeflate_deflate_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

data/ext/libdeflate/libdeflate/tools/afl-fuzz/deflate_decompress/inputs/0

@@ -0,0 +1,3 @@
+u�1
+�@Eg�Bl5
+��V���6j��X{�i=���l=����l�����?t���D

data/ext/libdeflate/libdeflate/tools/afl-fuzz/gzip_decompress/fuzz.c

@@ -0,0 +1,28 @@
+#include <assert.h>
+#include <libdeflate.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+int main(int argc, char **argv)
+{
+	struct libdeflate_decompressor *d;
+	int ret;
+	int fd = open(argv[1], O_RDONLY);
+	struct stat stbuf;
+	assert(fd >= 0);
+	ret = fstat(fd, &stbuf);
+	assert(!ret);
+
+	char in[stbuf.st_size];
+	ret = read(fd, in, sizeof in);
+	assert(ret == sizeof in);
+
+	char out[sizeof(in) * 3];
+
+	d = libdeflate_alloc_decompressor();
+
+	libdeflate_gzip_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

data/ext/libdeflate/libdeflate/tools/afl-fuzz/prepare_for_fuzz.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+make -C ../../ clean
+make clean
+AFL_HARDEN=1 make CC=afl-gcc -C ../../
+AFL_HARDEN=1 make CC=afl-gcc
+
+for dir in $(find . -mindepth 1 -maxdepth 1 -type d); do
+	rm -rf /tmp/$dir
+	cp -va $dir /tmp/$dir
+	mkdir -p /tmp/$dir/outputs
+done

data/ext/libdeflate/libdeflate/tools/afl-fuzz/zlib_decompress/fuzz.c

@@ -0,0 +1,28 @@
+#include <assert.h>
+#include <libdeflate.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+int main(int argc, char **argv)
+{
+	struct libdeflate_decompressor *d;
+	int ret;
+	int fd = open(argv[1], O_RDONLY);
+	struct stat stbuf;
+	assert(fd >= 0);
+	ret = fstat(fd, &stbuf);
+	assert(!ret);
+
+	char in[stbuf.st_size];
+	ret = read(fd, in, sizeof in);
+	assert(ret == sizeof in);
+
+	char out[sizeof(in) * 3];
+
+	d = libdeflate_alloc_decompressor();
+
+	libdeflate_zlib_decompress(d, in, sizeof in, out, sizeof out, NULL);
+	libdeflate_free_decompressor(d);
+	return 0;
+}

data/ext/libdeflate/libdeflate/tools/afl-fuzz/zlib_decompress/inputs/0

@@ -0,0 +1,3 @@
+x�u�1
+�@Eg�Bl5
+��V���6j��X{�i=���l=����l�����?t���D

data/ext/libdeflate/libdeflate/tools/android_build.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -eu
+
+ARCH="arm32"
+COMPILER="gcc"
+NDKDIR="/opt/android-ndk"
+DISABLE_NEON=
+
+usage() {
+	cat << EOF
+Usage: $0 [OPTION]... -- [BENCHMARK_PROGRAM_ARG]...
+Build the libdeflate test programs for Android
+
+  --arch=ARCH          Architecture: arm32|arm64 (default: $ARCH)
+  --compiler=COMPILER  Compiler: gcc|clang (default: $COMPILER)
+  --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
+  --disable-neon       Disable NEON instructions
+EOF
+}
+
+if ! options=$(getopt -o '' \
+	-l 'arch:,compiler:,ndkdir:,disable-neon,help' -- "$@"); then
+	usage
+	exit 1
+fi
+
+eval set -- "$options"
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+	--arch)
+		ARCH="$2"
+		shift
+		;;
+	--compiler)
+		COMPILER="$2"
+		shift
+		;;
+	--ndkdir)
+		NDKDIR="$2"
+		shift
+		;;
+	--disable-neon)
+		DISABLE_NEON=1
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Unknown option \"$1\""
+		usage
+		exit 1
+	esac
+	shift
+done
+
+CFLAGS="-fPIC -pie"
+
+case "$ARCH" in
+arm|arm32|aarch32)
+	GCC_TOOLCHAIN="arm-linux-androideabi-4.9"
+	CLANG_TARGET="armv7-none-linux-androideabi"
+	if [ -n "$DISABLE_NEON" ]; then
+		CFLAGS+=" -march=armv6"
+	else
+		CFLAGS+=" -march=armv7-a -mfpu=neon -mfloat-abi=softfp"
+	fi
+	CFLAGS+=" --sysroot=\"$NDKDIR/platforms/android-12/arch-arm\""
+	;;
+arm64|aarch64)
+	GCC_TOOLCHAIN="aarch64-linux-android-4.9"
+	CLANG_TARGET="aarch64-none-linux-android"
+	CFLAGS+=" --sysroot=\"$NDKDIR/platforms/android-21/arch-arm64\""
+	;;
+*)
+	echo 1>&2 "Unknown architecture: \"$ARCH\""
+	usage
+	exit 1
+esac
+
+case "$COMPILER" in
+gcc)
+	CC="\"$NDKDIR/toolchains/$GCC_TOOLCHAIN/prebuilt/linux-x86_64/bin/${GCC_TOOLCHAIN%-*}-gcc\""
+	CFLAGS+=" -pie"
+	;;
+clang)
+	CC="\"$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/clang\""
+	CFLAGS+=" -target \"$CLANG_TARGET\""
+	CFLAGS+=" -gcc-toolchain \"$NDKDIR/toolchains/$GCC_TOOLCHAIN/prebuilt/linux-x86_64\""
+	;;
+*)
+	echo 1>&2 "Unknown compiler: \"$COMPILER\""
+	usage
+	exit 1
+esac
+
+make -j$(grep -c processor /proc/cpuinfo) test_programs \
+	CC="$CC" CFLAGS="$CFLAGS"

data/ext/libdeflate/libdeflate/tools/checksum_benchmarks.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -eu
+
+if [ $# -eq 0 ]; then
+	FILE="$HOME/data/silesia"
+	echo "Using default FILE: $FILE"
+	echo
+elif [ $# -eq 1 ]; then
+	FILE="$1"
+else
+	echo "Usage: $0 [FILE]" 1>&2
+	exit 1
+fi
+
+
+if ! grep -q '\<sse2\>' /proc/cpuinfo; then
+	echo "This script must be run on an x86 CPU" 1>&2
+	exit 1
+fi
+
+if [ -n "$(git status -s | egrep 'adler32.c|crc32.c')" ]; then
+	echo "This script will overwrite adler32.c and crc32.c, which" \
+		"have uncommitted changes.  Refusing to run." 1>&2
+	exit 1
+fi
+
+do_benchmark() {
+	method="$1"
+	shift
+	speed=$(./checksum "$@" -t "$FILE" | \
+		grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+')
+	printf "%-36s%-10s\n" "$method" "$speed"
+}
+
+sort_by_speed() {
+	awk '{print $NF, $0}' | sort -nr | cut -f2- -d' '
+}
+
+cat << EOF
+Method                              Speed (MB/s)
+------                              ------------
+EOF
+
+# CRC-32
+(
+if grep -q '\<pclmulqdq\>' /proc/cpuinfo; then
+	make checksum > /dev/null
+	do_benchmark "CRC-32 (libdeflate, PCLMUL/AVX)"
+	sed -i '/^\#if NEED_PCLMUL_IMPL && !defined(__AVX__)/,/^\#endif$/d' lib/crc32.c
+	make checksum > /dev/null
+	do_benchmark "CRC-32 (libdeflate, PCLMUL)"
+fi
+sed -i '/^#if defined(__PCLMUL__)/,/^\#endif$/d' lib/crc32.c
+make checksum > /dev/null
+do_benchmark "CRC-32 (libdeflate, generic)"
+git checkout -f lib/crc32.c > /dev/null
+do_benchmark "CRC-32 (zlib)" -Z
+) | sort_by_speed
+
+# Adler-32
+echo
+(
+if grep -q '\<avx2\>' /proc/cpuinfo; then
+	make checksum > /dev/null
+	do_benchmark "Adler-32 (libdeflate, AVX2)" -A
+fi
+sed -i '/^#if defined(__AVX2__)/,/^\#endif$/d' lib/adler32.c
+make checksum > /dev/null
+do_benchmark "Adler-32 (libdeflate, SSE2)" -A
+sed -i '/^#ifdef __SSE2__/,/^\#endif$/d' lib/adler32.c
+make checksum > /dev/null
+do_benchmark "Adler-32 (libdeflate, generic)" -A
+git checkout -f lib/adler32.c > /dev/null
+do_benchmark "Adler-32 (zlib)" -A -Z
+) | sort_by_speed

data/ext/libdeflate/libdeflate/tools/exec_tests.sh

@@ -0,0 +1,30 @@
+#
+# Helper script used by run_tests.sh, not intended to be run directly
+#
+
+set -eu
+
+run_cmd() {
+	echo "$WRAPPER $@"
+	$WRAPPER "$@" > /dev/null
+}
+
+run_cmd ./test_checksums
+
+for format in '' '-g' '-z'; do
+	for ref_impl in '' '-Y' '-Z'; do
+		run_cmd ./benchmark $format $ref_impl $SMOKEDATA
+	done
+done
+for level in 1 3 7 9; do
+	for ref_impl in '' '-Y'; do
+		run_cmd ./benchmark -$level $ref_impl $SMOKEDATA
+	done
+done
+for level in 1 3 7 9 12; do
+	for ref_impl in '' '-Z'; do
+		run_cmd ./benchmark -$level $ref_impl $SMOKEDATA
+	done
+done
+
+echo "exec_tests finished successfully" # Needed for 'adb shell'

data/ext/libdeflate/libdeflate/tools/gen_crc32_multipliers.c

@@ -0,0 +1,108 @@
+/*
+ * gen_crc32_multipliers.c
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <inttypes.h>
+#include <stdio.h>
+
+/* generator polynomial G(x) */
+#define CRCPOLY		0xEDB88320 /* G(x) without x^32 term */
+#define CRCPOLY_FULL	(((uint64_t)CRCPOLY << 1) | 1) /* G(x) */
+
+/* Compute x^D mod G(x) */
+static uint32_t
+compute_multiplier(int D)
+{
+	/* Start with x^0 mod G(x) */
+	uint32_t remainder = 0x80000000;
+
+	/* Each iteration, 'remainder' becomes x^i mod G(x) */
+	for (int i = 1; i <= D; i++)
+		remainder = (remainder >> 1) ^ ((remainder & 1) ? CRCPOLY : 0);
+
+	/* Now 'remainder' is x^D mod G(x) */
+	return remainder;
+}
+
+/* Compute floor(x^64 / G(x)) */
+static uint64_t
+compute_barrett_reduction_constant(void)
+{
+	uint64_t quotient = 0;
+	uint64_t dividend = 0x1;
+
+	for (int i = 0; i < 64 - 32 + 1; i++) {
+		if ((dividend >> i) & 1) {
+			quotient |= (uint64_t)1 << i;
+			dividend ^= CRCPOLY_FULL << i;
+		}
+	}
+
+	return quotient;
+}
+
+/*
+ * This program computes the constant multipliers needed for carryless
+ * multiplication accelerated CRC-32.  It assumes 128-bit vectors divided into
+ * two 64-bit halves which are multiplied separately with different 32-bit
+ * multipliers, producing two 95-bit products.  For a given number of 128-bit
+ * vectors per iteration, the program outputs a pair of multipliers, one for
+ * each 64-bit half.
+ *
+ * Careful: all polynomials are "bit-reversed", meaning that the low-order bits
+ * have the highest degree and the high-order bits have the lowest degree!
+ */
+int
+main(void)
+{
+	printf("\t/* Constants precomputed by gen_crc32_multipliers.c.  "
+	       "Do not edit! */\n");
+
+	/* High and low multipliers for each needed vector count */
+	for (int order = 2; order >= 0; order--) {
+		int vecs_per_iteration = 1 << order;
+		int right = (128 * vecs_per_iteration) + 95;
+		printf("\tconst __v2di multipliers_%d = (__v2di)"
+		       "{ 0x%08"PRIX32", 0x%08"PRIX32" };\n",
+		       vecs_per_iteration,
+		       compute_multiplier(right - 64) /* higher degree half */,
+		       compute_multiplier(right - 128) /* lower degree half */);
+	}
+
+	/* Multiplier for final 96 => 64 bit fold */
+	printf("\tconst __v2di final_multiplier = (__v2di){ 0x%08"PRIX32" };\n",
+	       compute_multiplier(63));
+
+	/* 32-bit mask */
+	printf("\tconst __m128i mask32 = (__m128i)(__v4si){ 0xFFFFFFFF };\n");
+
+	/* Constants for final 64 => 32 bit reduction */
+	printf("\tconst __v2di barrett_reduction_constants =\n"
+	       "\t\t\t(__v2di){ 0x%016"PRIX64", 0x%016"PRIX64" };\n",
+	       compute_barrett_reduction_constant(), CRCPOLY_FULL);
+
+	return 0;
+}

data/ext/libdeflate/libdeflate/tools/gen_crc32_table.c

@@ -0,0 +1,100 @@
+/*
+ * gen_crc32_table.c - a program for CRC-32 table generation
+ *
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+
+static uint32_t crc32_table[0x800];
+
+static uint32_t
+crc32_update_bit(uint32_t remainder, uint8_t next_bit)
+{
+	return (remainder >> 1) ^
+		(((remainder ^ next_bit) & 1) ? 0xEDB88320 : 0);
+}
+
+static uint32_t
+crc32_update_byte(uint32_t remainder, uint8_t next_byte)
+{
+	for (int j = 0; j < 8; j++, next_byte >>= 1)
+		remainder = crc32_update_bit(remainder, next_byte & 1);
+	return remainder;
+}
+
+static void
+print_256_entries(const uint32_t *entries)
+{
+	for (size_t i = 0; i < 256 / 4; i++) {
+		printf("\t");
+		for (size_t j = 0; j < 4; j++) {
+			printf("0x%08x,", entries[i * 4 + j]);
+			if (j != 3)
+				printf(" ");
+		}
+		printf("\n");
+	}
+}
+
+int
+main(void)
+{
+	/* crc32_table[i] for 0 <= i < 0x100 is the CRC-32 of byte i.  */
+	for (int i = 0; i < 0x100; i++)
+		crc32_table[i] = crc32_update_byte(0, i);
+
+	/* crc32_table[i] for 0x100 <= i < 0x800 is the CRC-32 of byte i % 0x100
+	 * followed by i / 0x100 zero bytes.  */
+	for (int i = 0x100; i < 0x800; i++)
+		crc32_table[i] = crc32_update_byte(crc32_table[i - 0x100], 0);
+
+	printf("/*\n");
+	printf(" * crc32_table.h - data table to accelerate CRC-32 computation\n");
+	printf(" *\n");
+	printf(" * THIS FILE WAS AUTOMATICALLY GENERATED "
+	       "BY gen_crc32_table.c.  DO NOT EDIT.\n");
+	printf(" */\n");
+	printf("\n");
+	printf("#include <stdint.h>\n");
+	printf("\n");
+	printf("static const uint32_t crc32_table[] = {\n");
+	print_256_entries(&crc32_table[0x000]);
+	printf("#if defined(CRC32_SLICE4) || defined(CRC32_SLICE8)\n");
+	print_256_entries(&crc32_table[0x100]);
+	print_256_entries(&crc32_table[0x200]);
+	print_256_entries(&crc32_table[0x300]);
+	printf("#endif /* CRC32_SLICE4 || CRC32_SLICE8 */\n");
+	printf("#if defined(CRC32_SLICE8)\n");
+	print_256_entries(&crc32_table[0x400]);
+	print_256_entries(&crc32_table[0x500]);
+	print_256_entries(&crc32_table[0x600]);
+	print_256_entries(&crc32_table[0x700]);
+	printf("#endif /* CRC32_SLICE8 */\n");
+	printf("};\n");
+	return 0;
+}