zstd-native-ruby 1.0.0

data/vendor/zstd/tests/seqgen.c

@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "seqgen.h"
+#include "mem.h"
+#include <string.h>
+
+#define MIN(a, b)  ((a) < (b) ? (a) : (b))
+
+static const size_t kMatchBytes = 128;
+
+#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static BYTE SEQ_randByte(unsigned* src)
+{
+    static const U32 prime1 = 2654435761U;
+    static const U32 prime2 = 2246822519U;
+    U32 rand32 = *src;
+    rand32 *= prime1;
+    rand32 ^= prime2;
+    rand32  = SEQ_rotl32(rand32, 13);
+    *src = rand32;
+    return (BYTE)(rand32 >> 5);
+}
+
+SEQ_stream SEQ_initStream(unsigned seed)
+{
+    SEQ_stream stream;
+    stream.state = 0;
+    XXH64_reset(&stream.xxh, 0);
+    stream.seed = seed;
+    return stream;
+}
+
+/* Generates a single guard byte, then match length + 1 of a different byte,
+ * then another guard byte.
+ */
+static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value,
+                                  SEQ_outBuffer* out)
+{
+    typedef enum {
+        ml_first_byte = 0,
+        ml_match_bytes,
+        ml_last_byte,
+    } ml_state;
+    BYTE* const ostart = (BYTE*)out->dst;
+    BYTE* const oend = ostart + out->size;
+    BYTE* op = ostart + out->pos;
+
+    switch ((ml_state)stream->state) {
+    case ml_first_byte:
+        /* Generate a single byte and pick a different byte for the match */
+        if (op >= oend) {
+            stream->bytesLeft = 1;
+            break;
+        }
+        *op = SEQ_randByte(&stream->seed) & 0xFF;
+        do {
+            stream->saved = SEQ_randByte(&stream->seed) & 0xFF;
+        } while (*op == stream->saved);
+        ++op;
+        /* State transition */
+        stream->state = ml_match_bytes;
+        stream->bytesLeft = value + 1;
+    /* fall-through */
+    case ml_match_bytes: {
+        /* Copy matchLength + 1 bytes to the output buffer */
+        size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
+        if (setLength > 0) {
+            memset(op, stream->saved, setLength);
+            op += setLength;
+            stream->bytesLeft -= setLength;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = ml_last_byte;
+    }
+    /* fall-through */
+    case ml_last_byte:
+        /* Generate a single byte and pick a different byte for the match */
+        if (op >= oend) {
+            stream->bytesLeft = 1;
+            break;
+        }
+        do {
+            *op = SEQ_randByte(&stream->seed) & 0xFF;
+        } while (*op == stream->saved);
+        ++op;
+        /* State transition */
+    /* fall-through */
+    default:
+        stream->state = 0;
+        stream->bytesLeft = 0;
+        break;
+    }
+    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+    out->pos = op - ostart;
+    return stream->bytesLeft;
+}
+
+/* Saves the current seed then generates kMatchBytes random bytes >= 128.
+ * Generates literal length - kMatchBytes random bytes < 128.
+ * Generates another kMatchBytes using the saved seed to generate a match.
+ * This way the match is easy to find for the compressors.
+ */
+static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
+{
+    typedef enum {
+        ll_start = 0,
+        ll_run_bytes,
+        ll_literals,
+        ll_run_match,
+    } ll_state;
+    BYTE* const ostart = (BYTE*)out->dst;
+    BYTE* const oend = ostart + out->size;
+    BYTE* op = ostart + out->pos;
+
+    switch ((ll_state)stream->state) {
+    case ll_start:
+        stream->state = ll_run_bytes;
+        stream->saved = stream->seed;
+        stream->bytesLeft = MIN(kMatchBytes, value);
+    /* fall-through */
+    case ll_run_bytes:
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->seed) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = ll_literals;
+        stream->bytesLeft = value - MIN(kMatchBytes, value);
+    /* fall-through */
+    case ll_literals:
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->seed) & 0x7F;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = ll_run_match;
+        stream->bytesLeft = MIN(kMatchBytes, value);
+    /* fall-through */
+    case ll_run_match: {
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->saved) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+    }
+    /* fall-through */
+    default:
+        stream->state = 0;
+        stream->bytesLeft = 0;
+        break;
+    }
+    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+    out->pos = op - ostart;
+    return stream->bytesLeft;
+}
+
+/* Saves the current seed then generates kMatchBytes random bytes >= 128.
+ * Generates offset - kMatchBytes of zeros to get a large offset without
+ * polluting the hash tables.
+ * Generates another kMatchBytes using the saved seed to generate a with the
+ * required offset.
+ */
+static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
+{
+    typedef enum {
+        of_start = 0,
+        of_run_bytes,
+        of_offset,
+        of_run_match,
+    } of_state;
+    BYTE* const ostart = (BYTE*)out->dst;
+    BYTE* const oend = ostart + out->size;
+    BYTE* op = ostart + out->pos;
+
+    switch ((of_state)stream->state) {
+    case of_start:
+        stream->state = of_run_bytes;
+        stream->saved = stream->seed;
+        stream->bytesLeft = MIN(value, kMatchBytes);
+    /* fall-through */
+    case of_run_bytes: {
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->seed) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = of_offset;
+        stream->bytesLeft = value - MIN(value, kMatchBytes);
+    }
+    /* fall-through */
+    case of_offset: {
+        /* Copy matchLength + 1 bytes to the output buffer */
+        size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
+        if (setLength > 0) {
+            memset(op, 0, setLength);
+            op += setLength;
+            stream->bytesLeft -= setLength;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+        /* State transition */
+        stream->state = of_run_match;
+        stream->bytesLeft = MIN(value, kMatchBytes);
+    }
+    /* fall-through */
+    case of_run_match: {
+        while (stream->bytesLeft > 0 && op < oend) {
+            *op++ = SEQ_randByte(&stream->saved) | 0x80;
+            --stream->bytesLeft;
+        }
+        if (stream->bytesLeft > 0)
+            break;
+    }
+    /* fall-through */
+    default:
+        stream->state = 0;
+        stream->bytesLeft = 0;
+        break;
+    }
+    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+    out->pos = op - ostart;
+    return stream->bytesLeft;
+}
+
+/* Returns the number of bytes left to generate.
+ * Must pass the same type/value until it returns 0.
+ */
+size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out)
+{
+    switch (type) {
+        case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out);
+        case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out);
+        case SEQ_gen_of: return SEQ_gen_offset(stream, value, out);
+        case SEQ_gen_max: /* fall-through */
+        default: return 0;
+    }
+}
+
+/* Returns the xxhash of the data produced so far */
+XXH64_hash_t SEQ_digest(SEQ_stream const* stream)
+{
+    return XXH64_digest(&stream->xxh);
+}

data/vendor/zstd/tests/seqgen.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef SEQGEN_H
+#define SEQGEN_H
+
+#define XXH_STATIC_LINKING_ONLY
+
+#include "xxhash.h"
+#include <stddef.h>   /* size_t */
+
+typedef enum {
+    SEQ_gen_ml = 0,
+    SEQ_gen_ll,
+    SEQ_gen_of,
+    SEQ_gen_max /* Must be the last value */
+} SEQ_gen_type;
+
+/* Internal state, do not use */
+typedef struct {
+    XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */
+    unsigned seed;
+    int state; /* enum to control state machine (clean=0) */
+    unsigned saved;
+    size_t bytesLeft;
+} SEQ_stream;
+
+SEQ_stream SEQ_initStream(unsigned seed);
+
+typedef struct {
+    void* dst;
+    size_t size;
+    size_t pos;
+} SEQ_outBuffer;
+
+/* Returns non-zero until the current type/value has been generated.
+ * Must pass the same type/value until it returns 0.
+ *
+ * Recommended to pick a value in the middle of the range you want, since there
+ * may be some noise that causes actual results to be slightly different.
+ * We try to be more accurate for smaller values.
+ *
+ * NOTE: Very small values don't work well (< 6).
+ */
+size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value,
+               SEQ_outBuffer* out);
+
+/* Returns the xxhash of the data produced so far */
+XXH64_hash_t SEQ_digest(SEQ_stream const* stream);
+
+#endif /* SEQGEN_H */

data/vendor/zstd/tests/test-license.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+import enum
+import glob
+import os
+import re
+import sys
+
+ROOT = os.path.join(os.path.dirname(__file__), "..")
+
+RELDIRS = [
+    "doc",
+    "examples",
+    "lib",
+    "programs",
+    "tests",
+    "contrib/linux-kernel",
+]
+
+REL_EXCLUDES = [
+    "contrib/linux-kernel/test/include",
+]
+
+def to_abs(d):
+    return os.path.normpath(os.path.join(ROOT, d)) + "/"
+
+DIRS = [to_abs(d) for d in RELDIRS]
+EXCLUDES = [to_abs(d) for d in REL_EXCLUDES]
+
+SUFFIXES = [
+    ".c",
+    ".h",
+    "Makefile",
+    ".mk",
+    ".py",
+    ".S",
+]
+
+# License should certainly be in the first 10 KB.
+MAX_BYTES = 10000
+MAX_LINES = 50
+
+LICENSE_LINES = [
+    "This source code is licensed under both the BSD-style license (found in the",
+    "LICENSE file in the root directory of this source tree) and the GPLv2 (found",
+    "in the COPYING file in the root directory of this source tree).",
+    "You may select, at your option, one of the above-listed licenses.",
+]
+
+COPYRIGHT_EXCEPTIONS = {
+    # From zstdmt
+    "threading.c",
+    "threading.h",
+    # From divsufsort
+    "divsufsort.c",
+    "divsufsort.h",
+}
+
+LICENSE_EXCEPTIONS = {
+    # From divsufsort
+    "divsufsort.c",
+    "divsufsort.h",
+    # License is slightly different because it references GitHub
+    "linux_zstd.h",
+}
+
+
+def valid_copyright(lines):
+    YEAR_REGEX = re.compile("\d\d\d\d|present")
+    for line in lines:
+        line = line.strip()
+        if "Copyright" not in line:
+            continue
+        if "present" in line:
+            return (False, f"Copyright line '{line}' contains 'present'!")
+        if "Meta Platforms, Inc" not in line:
+            return (False, f"Copyright line '{line}' does not contain 'Meta Platforms, Inc'")
+        year = YEAR_REGEX.search(line)
+        if year is not None:
+            return (False, f"Copyright line '{line}' contains {year.group(0)}; it should be yearless")
+        if " (c) " not in line:
+            return (False, f"Copyright line '{line}' does not contain ' (c) '!")
+        return (True, "")
+    return (False, "Copyright not found!")
+
+
+def valid_license(lines):
+    for b in range(len(lines)):
+        if LICENSE_LINES[0] not in lines[b]:
+            continue
+        for l in range(len(LICENSE_LINES)):
+            if LICENSE_LINES[l] not in lines[b + l]:
+                message = f"""Invalid license line found starting on line {b + l}!
+Expected: '{LICENSE_LINES[l]}'
+Actual: '{lines[b + l]}'"""
+                return (False, message)
+        return (True, "")
+    return (False, "License not found!")
+
+
+def valid_file(filename):
+    with open(filename, "r") as f:
+        lines = f.readlines(MAX_BYTES)
+    lines = lines[:min(len(lines), MAX_LINES)]
+
+    ok = True
+    if os.path.basename(filename) not in COPYRIGHT_EXCEPTIONS:
+        c_ok, c_msg = valid_copyright(lines)
+        if not c_ok:
+            print(f"{filename}: {c_msg}", file=sys.stderr)
+            ok = False
+    if os.path.basename(filename) not in LICENSE_EXCEPTIONS:
+        l_ok, l_msg = valid_license(lines)
+        if not l_ok:
+            print(f"{filename}: {l_msg}", file=sys.stderr)
+            ok = False
+    return ok
+
+
+def exclude(filename):
+    for x in EXCLUDES:
+        if filename.startswith(x):
+            return True
+    return False
+
+def main():
+    invalid_files = []
+    for directory in DIRS:
+        for suffix in SUFFIXES:
+            files = set(glob.glob(f"{directory}/**/*{suffix}", recursive=True))
+            for filename in files:
+                if exclude(filename):
+                    continue
+                if not valid_file(filename):
+                    invalid_files.append(filename)
+    if len(invalid_files) > 0:
+        print("Fail!", file=sys.stderr)
+        for f in invalid_files:
+            print(f)
+        return 1
+    else:
+        print("Pass!", file=sys.stderr)
+        return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

data/vendor/zstd/tests/test-variants.sh

@@ -0,0 +1,115 @@
+#!/bin/sh
+set -e
+set -u
+set -x
+
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+PROG_DIR="$SCRIPT_DIR/../programs"
+
+ZSTD="$PROG_DIR/zstd"
+ZSTD_COMPRESS="$PROG_DIR/zstd-compress"
+ZSTD_DECOMPRESS="$PROG_DIR/zstd-decompress"
+ZSTD_NOLEGACY="$PROG_DIR/zstd-nolegacy"
+ZSTD_DICTBUILDER="$PROG_DIR/zstd-dictBuilder"
+ZSTD_FRUGAL="$PROG_DIR/zstd-frugal"
+ZSTD_NOMT="$PROG_DIR/zstd-nomt"
+
+println() {
+    printf '%b\n' "${*}"
+}
+
+die() {
+    println "$@" 1>&2
+    exit 1
+}
+
+symbol_present() {
+	(nm $1 || echo "symbol_present $@ failed") | grep $2
+}
+
+symbol_not_present() {
+	symbol_present $@ && die "Binary '$1' mistakenly contains symbol '$2'" ||:
+}
+
+compress_not_present() {
+	symbol_not_present "$1" ZSTD_compress
+}
+
+decompress_not_present() {
+	symbol_not_present "$1" ZSTD_decompress
+}
+
+dict_not_present() {
+	symbol_not_present "$1" ZDICT_
+	symbol_not_present "$1" COVER_
+}
+
+cliextra_not_present() {
+	symbol_not_present "$1" TRACE_
+	symbol_not_present "$1" BMK_
+}
+
+legacy_not_present() {
+	symbol_not_present "$1" ZSTDv0
+}
+
+test_help() {
+	"$1" --help | grep -- "$2"
+}
+
+test_no_help() {
+	test_help $@ && die "'$1' supports '$2' when it shouldn't" ||:
+}
+
+extras_not_present() {
+	dict_not_present $@
+	legacy_not_present $@
+	cliextra_not_present $@
+	test_no_help $@ "--train"
+	test_no_help $@ "-b#"
+}
+
+test_compress() {
+	echo "hello" | "$1" | "$ZSTD" -t
+}
+
+test_decompress() {
+	echo "hello" | "$ZSTD" | "$1" -t
+}
+
+test_zstd() {
+	test_compress $@
+	test_decompress $@
+}
+
+extras_not_present "$ZSTD_FRUGAL"
+extras_not_present "$ZSTD_COMPRESS"
+extras_not_present "$ZSTD_DECOMPRESS"
+
+compress_not_present "$ZSTD_DECOMPRESS"
+
+decompress_not_present "$ZSTD_COMPRESS"
+decompress_not_present "$ZSTD_DICTBUILDER"
+
+cliextra_not_present "$ZSTD_DICTBUILDER"
+
+legacy_not_present "$ZSTD_DICTBUILDER"
+legacy_not_present "$ZSTD_NOLEGACY"
+
+symbol_not_present "$ZSTD" ZSTDv01
+symbol_not_present "$ZSTD" ZSTDv02
+symbol_not_present "$ZSTD" ZSTDv03
+symbol_not_present "$ZSTD" ZSTDv04
+
+test_compress "$ZSTD_COMPRESS"
+test_decompress "$ZSTD_DECOMPRESS"
+
+test_zstd "$ZSTD_FRUGAL"
+test_zstd "$ZSTD_NOLEGACY"
+
+test_help "$ZSTD" '-b#'
+test_help "$ZSTD" --train
+test_help "$ZSTD_DICTBUILDER" --train
+
+println "Success!"