deflate-ruby 0.1.0

data/ext/deflate_ruby/libdeflate/programs/test_util.c

@@ -0,0 +1,237 @@
+/*
+ * test_util.c - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "test_util.h"
+
+#include <fcntl.h>
+#include <time.h>
+#ifdef _WIN32
+#  include <windows.h>
+#else
+#  include <unistd.h>
+#  include <sys/mman.h>
+#  include <sys/time.h>
+#endif
+
+#ifndef MAP_ANONYMOUS
+#  define MAP_ANONYMOUS MAP_ANON
+#endif
+
+/* Abort with an error message */
+NORETURN void
+assertion_failed(const char *expr, const char *file, int line)
+{
+	msg("Assertion failed: %s at %s:%d", expr, file, line);
+	abort();
+}
+
+void
+begin_performance_test(void)
+{
+	/* Skip performance tests by default, since they can be flaky. */
+	if (getenv("INCLUDE_PERF_TESTS") == NULL)
+		exit(0);
+}
+
+static size_t
+get_page_size(void)
+{
+#ifdef _WIN32
+	SYSTEM_INFO info;
+
+	GetSystemInfo(&info);
+	return info.dwPageSize;
+#else
+	return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+/* Allocate a buffer with guard pages */
+void
+alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret)
+{
+	const size_t pagesize = get_page_size();
+	const size_t nr_pages = (size + pagesize - 1) / pagesize;
+	u8 *base_addr;
+	u8 *start, *end;
+#ifdef _WIN32
+	DWORD oldProtect;
+#endif
+
+	*start_ret = NULL;
+	*end_ret = NULL;
+
+#ifdef _WIN32
+	/* Allocate buffer and guard pages with no access. */
+	base_addr = VirtualAlloc(NULL, (nr_pages + 2) * pagesize,
+				 MEM_COMMIT | MEM_RESERVE, PAGE_NOACCESS);
+	if (!base_addr) {
+		msg("Unable to allocate memory (VirtualAlloc): Windows error %u",
+		    (unsigned int)GetLastError());
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Grant read+write access to just the buffer. */
+	if (!VirtualProtect(start, end - start, PAGE_READWRITE, &oldProtect)) {
+		msg("Unable to protect memory (VirtualProtect): Windows error %u",
+		    (unsigned int)GetLastError());
+		VirtualFree(base_addr, 0, MEM_RELEASE);
+		ASSERT(0);
+	}
+#else
+	/* Allocate buffer and guard pages. */
+	base_addr = mmap(NULL, (nr_pages + 2) * pagesize, PROT_READ|PROT_WRITE,
+			 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	if (base_addr == (u8 *)MAP_FAILED) {
+		msg_errno("Unable to allocate memory (anonymous mmap)");
+		ASSERT(0);
+	}
+	start = base_addr + pagesize;
+	end = start + (nr_pages * pagesize);
+
+	/* Unmap the guard pages. */
+	munmap(base_addr, pagesize);
+	munmap(end, pagesize);
+#endif
+	*start_ret = start;
+	*end_ret = end;
+}
+
+/* Free a buffer that was allocated by alloc_guarded_buffer() */
+void
+free_guarded_buffer(u8 *start, u8 *end)
+{
+	if (!start)
+		return;
+#ifdef _WIN32
+	VirtualFree(start - get_page_size(), 0, MEM_RELEASE);
+#else
+	munmap(start, end - start);
+#endif
+}
+
+/*
+ * Return the number of timer ticks that have elapsed since some unspecified
+ * point fixed at the start of program execution
+ */
+u64
+timer_ticks(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER count;
+
+	QueryPerformanceCounter(&count);
+	return count.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME) || \
+	/* fallback detection method for direct compilation */ \
+	(!defined(HAVE_CONFIG_H) && defined(CLOCK_MONOTONIC))
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (1000000000 * (u64)ts.tv_sec) + ts.tv_nsec;
+#else
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (1000000 * (u64)tv.tv_sec) + tv.tv_usec;
+#endif
+}
+
+/*
+ * Return the number of timer ticks per second
+ */
+static u64
+timer_frequency(void)
+{
+#ifdef _WIN32
+	LARGE_INTEGER freq;
+
+	QueryPerformanceFrequency(&freq);
+	return freq.QuadPart;
+#elif defined(HAVE_CLOCK_GETTIME) || \
+	/* fallback detection method for direct compilation */ \
+	(!defined(HAVE_CONFIG_H) && defined(CLOCK_MONOTONIC))
+	return 1000000000;
+#else
+	return 1000000;
+#endif
+}
+
+/*
+ * Convert a number of elapsed timer ticks to milliseconds
+ */
+u64 timer_ticks_to_ms(u64 ticks)
+{
+	return ticks * 1000 / timer_frequency();
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to MB/s
+ */
+u64 timer_MB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000000;
+}
+
+/*
+ * Convert a byte count and a number of elapsed timer ticks to KB/s
+ */
+u64 timer_KB_per_s(u64 bytes, u64 ticks)
+{
+	return bytes * timer_frequency() / ticks / 1000;
+}
+
+bool
+put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits)
+{
+	os->bitbuf |= bits << os->bitcount;
+	os->bitcount += num_bits;
+	while (os->bitcount >= 8) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	return true;
+}
+
+bool
+flush_bits(struct output_bitstream *os)
+{
+	while (os->bitcount > 0) {
+		if (os->next == os->end)
+			return false;
+		*os->next++ = os->bitbuf;
+		os->bitcount -= 8;
+		os->bitbuf >>= 8;
+	}
+	os->bitcount = 0;
+	return true;
+}

data/ext/deflate_ruby/libdeflate/programs/test_util.h

@@ -0,0 +1,61 @@
+/*
+ * test_util.h - utility functions for test programs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PROGRAMS_TEST_UTIL_H
+#define PROGRAMS_TEST_UTIL_H
+
+#include "prog_util.h" /* must be included first */
+
+#include <zlib.h> /* for comparison purposes */
+
+NORETURN void
+assertion_failed(const char *expr, const char *file, int line);
+
+#define ASSERT(expr) { if (unlikely(!(expr))) \
+	assertion_failed(#expr, __FILE__, __LINE__); }
+
+void begin_performance_test(void);
+
+void alloc_guarded_buffer(size_t size, u8 **start_ret, u8 **end_ret);
+void free_guarded_buffer(u8 *start, u8 *end);
+
+u64 timer_ticks(void);
+u64 timer_ticks_to_ms(u64 ticks);
+u64 timer_MB_per_s(u64 bytes, u64 ticks);
+u64 timer_KB_per_s(u64 bytes, u64 ticks);
+
+struct output_bitstream {
+	machine_word_t bitbuf;
+	int bitcount;
+	u8 *next;
+	u8 *end;
+};
+
+bool put_bits(struct output_bitstream *os, machine_word_t bits, int num_bits);
+bool flush_bits(struct output_bitstream *os);
+
+#endif /* PROGRAMS_TEST_UTIL_H */

data/ext/deflate_ruby/libdeflate/programs/tgetopt.c

@@ -0,0 +1,118 @@
+/*
+ * tgetopt.c - portable replacement for GNU getopt()
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "prog_util.h"
+
+tchar *toptarg;
+int toptind = 1, topterr = 1, toptopt;
+
+/*
+ * This is a simple implementation of getopt().  It can be compiled with either
+ * 'char' or 'wchar_t' as the character type.
+ *
+ * Do *not* use this implementation if you need any of the following features,
+ * as they are not supported:
+ *	- Long options
+ *	- Option-related arguments retained in argv, not nulled out
+ *	- '+' and '-' characters in optstring
+ */
+int
+tgetopt(int argc, tchar *argv[], const tchar *optstring)
+{
+	static tchar empty[1];
+	static tchar *nextchar;
+	static bool done;
+
+	if (toptind == 1) {
+		/* Starting to scan a new argument vector */
+		nextchar = NULL;
+		done = false;
+	}
+
+	while (!done && (nextchar != NULL || toptind < argc)) {
+		if (nextchar == NULL) {
+			/* Scanning a new argument */
+			tchar *arg = argv[toptind++];
+			if (arg[0] == '-' && arg[1] != '\0') {
+				if (arg[1] == '-' && arg[2] == '\0') {
+					/* All args after "--" are nonoptions */
+					argv[toptind - 1] = NULL;
+					done = true;
+				} else {
+					/* Start of short option characters */
+					nextchar = &arg[1];
+				}
+			}
+		} else {
+			/* More short options in previous arg */
+			tchar opt = *nextchar;
+			tchar *p = tstrchr(optstring, opt);
+			if (p == NULL) {
+				if (topterr)
+					msg("invalid option -- '%"TC"'", opt);
+				toptopt = opt;
+				return '?';
+			}
+			/* 'opt' is a valid short option character */
+			nextchar++;
+			toptarg = NULL;
+			if (*(p + 1) == ':') {
+				/* 'opt' can take an argument */
+				if (*nextchar != '\0') {
+					/* Optarg is in same argv argument */
+					toptarg = nextchar;
+					nextchar = empty;
+				} else if (toptind < argc && *(p + 2) != ':') {
+					/* Optarg is next argv argument */
+					argv[toptind - 1] = NULL;
+					toptarg = argv[toptind++];
+				} else if (*(p + 2) != ':') {
+					if (topterr && *optstring != ':') {
+						msg("option requires an "
+						    "argument -- '%"TC"'", opt);
+					}
+					toptopt = opt;
+					opt = (*optstring == ':') ? ':' : '?';
+				}
+			}
+			if (*nextchar == '\0') {
+				argv[toptind - 1] = NULL;
+				nextchar = NULL;
+			}
+			return opt;
+		}
+	}
+
+	/* Done scanning.  Move all nonoptions to the end, set optind to the
+	 * index of the first nonoption, and return -1. */
+	toptind = argc;
+	while (--argc > 0)
+		if (argv[argc] != NULL)
+			argv[--toptind] = argv[argc];
+	done = true;
+	return -1;
+}

data/ext/deflate_ruby/libdeflate/scripts/android_build.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+SCRIPTDIR="$(dirname "$0")"
+BUILDDIR="$SCRIPTDIR/../build"
+API_LEVEL=28
+ARCH=arm64
+CFLAGS=${CFLAGS:-}
+ENABLE_CRC=false
+ENABLE_CRYPTO=false
+NDKDIR=$HOME/android-ndk-r25b
+
+usage() {
+	cat << EOF
+Usage: $0 [OPTION]...
+Build libdeflate for Android.
+
+  --api-level=LEVEL    Android API level to target (default: $API_LEVEL)
+  --arch=ARCH          Architecture: arm32|arm64|x86|x86_64 (default: $ARCH)
+  --enable-crc         Enable crc instructions
+  --enable-crypto      Enable crypto instructions
+  --ndkdir=NDKDIR      Android NDK directory (default: $NDKDIR)
+EOF
+}
+if ! options=$(getopt -o '' \
+	-l 'api-level:,arch:,enable-crc,enable-crypto,help,ndkdir:' -- "$@"); then
+	usage 1>&2
+	exit 1
+fi
+
+eval set -- "$options"
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+	--api-level)
+		API_LEVEL="$2"
+		shift
+		;;
+	--arch)
+		ARCH="$2"
+		shift
+		;;
+	--enable-crc)
+		ENABLE_CRC=true
+		;;
+	--enable-crypto)
+		ENABLE_CRYPTO=true
+		;;
+	--help)
+		usage
+		exit 0
+		;;
+	--ndkdir)
+		NDKDIR="$2"
+		shift
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo 1>&2 "Unknown option \"$1\""
+		usage 1>&2
+		exit 1
+	esac
+	shift
+done
+
+case "$ARCH" in
+arm|arm32|aarch32|armeabi-v7a)
+	ANDROID_ABI=armeabi-v7a
+	if $ENABLE_CRC || $ENABLE_CRYPTO; then
+		CFLAGS+=" -march=armv8-a"
+		if $ENABLE_CRC; then
+			CFLAGS+=" -mcrc"
+		else
+			CFLAGS+=" -mnocrc"
+		fi
+		if $ENABLE_CRYPTO; then
+			CFLAGS+=" -mfpu=crypto-neon-fp-armv8"
+		else
+			CFLAGS+=" -mfpu=neon"
+		fi
+	fi
+	;;
+arm64|aarch64|arm64-v8a)
+	ANDROID_ABI=arm64-v8a
+	features=""
+	if $ENABLE_CRC; then
+		features+="+crc"
+	fi
+	if $ENABLE_CRYPTO; then
+		features+="+crypto"
+	fi
+	if [ -n "$features" ]; then
+		CFLAGS+=" -march=armv8-a$features"
+	fi
+	;;
+x86)
+	ANDROID_ABI=x86
+	;;
+x86_64)
+	ANDROID_ABI=x86_64
+	;;
+*)
+	echo 1>&2 "Unknown architecture: \"$ARCH\""
+	usage 1>&2
+	exit 1
+esac
+
+"$SCRIPTDIR"/cmake-helper.sh -G Ninja \
+	-DCMAKE_TOOLCHAIN_FILE="$NDKDIR"/build/cmake/android.toolchain.cmake \
+	-DCMAKE_C_FLAGS="$CFLAGS" \
+	-DANDROID_ABI="$ANDROID_ABI" \
+	-DANDROID_PLATFORM="$API_LEVEL" \
+	-DLIBDEFLATE_BUILD_TESTS=1
+cmake --build "$BUILDDIR"

data/ext/deflate_ruby/libdeflate/scripts/android_tests.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Test libdeflate on a connected arm64 Android device.
+# Requires the Android NDK (release 19 or later) and adb.
+
+set -eu -o pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -ne 0 ]; then
+	echo 1>&2 "Usage: $0"
+	exit 2
+fi
+
+# Use NDKDIR if specified in environment, else use default value.
+: "${NDKDIR:=$HOME/android-ndk-r25b}"
+if [ ! -e "$NDKDIR" ]; then
+	cat 1>&2 << EOF
+Android NDK was not found in NDKDIR=$NDKDIR!  Set the
+environmental variable NDKDIR to the location of your Android NDK installation.
+EOF
+	exit 1
+fi
+
+CLEANUP_CMDS=()
+cleanup() {
+	for cmd in "${CLEANUP_CMDS[@]}"; do
+		eval "$cmd"
+	done
+}
+trap cleanup EXIT
+
+# Use TESTDATA if specified in environment, else generate it.
+if [ -z "${TESTDATA:-}" ]; then
+	# Generate default TESTDATA file.
+	TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX)
+	export TESTDATA
+	CLEANUP_CMDS+=("rm -f '$TESTDATA'")
+	find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \
+		-exec cat '{}' ';' | head -c 1000000 > "$TESTDATA"
+fi
+
+TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX)
+CLEANUP_CMDS+=("rm -r '$TMPDIR'")
+
+android_build_and_test() {
+	echo "Running Android tests with $*"
+
+	./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" > /dev/null
+	adb push "$TESTDATA" ./scripts/exec_tests.sh \
+		./build/programs/{benchmark,test_*} /data/local/tmp/ > /dev/null
+
+	# Note: adb shell always returns 0, even if the shell command fails...
+	adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \
+		> "$TMPDIR/adb.out"
+	if ! grep -q "exec_tests finished successfully" "$TMPDIR/adb.out"; then
+		echo 1>&2 "Android test failure!  adb shell output:"
+		cat "$TMPDIR/adb.out"
+		exit 1
+	fi
+}
+
+android_build_and_test --arch=arm32
+android_build_and_test --arch=arm32 --enable-crc
+android_build_and_test --arch=arm64
+android_build_and_test --arch=arm64 --enable-crc
+android_build_and_test --arch=arm64 --enable-crypto
+android_build_and_test --arch=arm64 --enable-crc --enable-crypto
+
+echo "Android tests passed"

data/ext/deflate_ruby/libdeflate/scripts/benchmark.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTDIR="$(dirname "$(realpath "$0")")"
+BUILDDIR="$SCRIPTDIR/../build"
+
+"$SCRIPTDIR"/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 -G Ninja > /dev/null
+ninja -C "$BUILDDIR" --quiet benchmark
+"$BUILDDIR"/programs/benchmark "$@"

data/ext/deflate_ruby/libdeflate/scripts/checksum.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTDIR="$(dirname "$(realpath "$0")")"
+BUILDDIR="$SCRIPTDIR/../build"
+
+"$SCRIPTDIR"/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 -G Ninja > /dev/null
+ninja -C "$BUILDDIR" --quiet checksum
+"$BUILDDIR"/programs/checksum "$@"