electrobun 0.0.1 → 0.0.2

package/src/bsdiff/zstd/tests/seqgen.c

@@ -1,260 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#include "seqgen.h"
-#include "mem.h"
-#include <string.h>
-
-#define MIN(a, b)  ((a) < (b) ? (a) : (b))
-
-static const size_t kMatchBytes = 128;
-
-#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-static BYTE SEQ_randByte(unsigned* src)
-{
-    static const U32 prime1 = 2654435761U;
-    static const U32 prime2 = 2246822519U;
-    U32 rand32 = *src;
-    rand32 *= prime1;
-    rand32 ^= prime2;
-    rand32  = SEQ_rotl32(rand32, 13);
-    *src = rand32;
-    return (BYTE)(rand32 >> 5);
-}
-
-SEQ_stream SEQ_initStream(unsigned seed)
-{
-    SEQ_stream stream;
-    stream.state = 0;
-    XXH64_reset(&stream.xxh, 0);
-    stream.seed = seed;
-    return stream;
-}
-
-/* Generates a single guard byte, then match length + 1 of a different byte,
- * then another guard byte.
- */
-static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value,
-                                  SEQ_outBuffer* out)
-{
-    typedef enum {
-        ml_first_byte = 0,
-        ml_match_bytes,
-        ml_last_byte,
-    } ml_state;
-    BYTE* const ostart = (BYTE*)out->dst;
-    BYTE* const oend = ostart + out->size;
-    BYTE* op = ostart + out->pos;
-
-    switch ((ml_state)stream->state) {
-    case ml_first_byte:
-        /* Generate a single byte and pick a different byte for the match */
-        if (op >= oend) {
-            stream->bytesLeft = 1;
-            break;
-        }
-        *op = SEQ_randByte(&stream->seed) & 0xFF;
-        do {
-            stream->saved = SEQ_randByte(&stream->seed) & 0xFF;
-        } while (*op == stream->saved);
-        ++op;
-        /* State transition */
-        stream->state = ml_match_bytes;
-        stream->bytesLeft = value + 1;
-    /* fall-through */
-    case ml_match_bytes: {
-        /* Copy matchLength + 1 bytes to the output buffer */
-        size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
-        if (setLength > 0) {
-            memset(op, stream->saved, setLength);
-            op += setLength;
-            stream->bytesLeft -= setLength;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-        /* State transition */
-        stream->state = ml_last_byte;
-    }
-    /* fall-through */
-    case ml_last_byte:
-        /* Generate a single byte and pick a different byte for the match */
-        if (op >= oend) {
-            stream->bytesLeft = 1;
-            break;
-        }
-        do {
-            *op = SEQ_randByte(&stream->seed) & 0xFF;
-        } while (*op == stream->saved);
-        ++op;
-        /* State transition */
-    /* fall-through */
-    default:
-        stream->state = 0;
-        stream->bytesLeft = 0;
-        break;
-    }
-    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
-    out->pos = op - ostart;
-    return stream->bytesLeft;
-}
-
-/* Saves the current seed then generates kMatchBytes random bytes >= 128.
- * Generates literal length - kMatchBytes random bytes < 128.
- * Generates another kMatchBytes using the saved seed to generate a match.
- * This way the match is easy to find for the compressors.
- */
-static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
-{
-    typedef enum {
-        ll_start = 0,
-        ll_run_bytes,
-        ll_literals,
-        ll_run_match,
-    } ll_state;
-    BYTE* const ostart = (BYTE*)out->dst;
-    BYTE* const oend = ostart + out->size;
-    BYTE* op = ostart + out->pos;
-
-    switch ((ll_state)stream->state) {
-    case ll_start:
-        stream->state = ll_run_bytes;
-        stream->saved = stream->seed;
-        stream->bytesLeft = MIN(kMatchBytes, value);
-    /* fall-through */
-    case ll_run_bytes:
-        while (stream->bytesLeft > 0 && op < oend) {
-            *op++ = SEQ_randByte(&stream->seed) | 0x80;
-            --stream->bytesLeft;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-        /* State transition */
-        stream->state = ll_literals;
-        stream->bytesLeft = value - MIN(kMatchBytes, value);
-    /* fall-through */
-    case ll_literals:
-        while (stream->bytesLeft > 0 && op < oend) {
-            *op++ = SEQ_randByte(&stream->seed) & 0x7F;
-            --stream->bytesLeft;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-        /* State transition */
-        stream->state = ll_run_match;
-        stream->bytesLeft = MIN(kMatchBytes, value);
-    /* fall-through */
-    case ll_run_match: {
-        while (stream->bytesLeft > 0 && op < oend) {
-            *op++ = SEQ_randByte(&stream->saved) | 0x80;
-            --stream->bytesLeft;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-    }
-    /* fall-through */
-    default:
-        stream->state = 0;
-        stream->bytesLeft = 0;
-        break;
-    }
-    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
-    out->pos = op - ostart;
-    return stream->bytesLeft;
-}
-
-/* Saves the current seed then generates kMatchBytes random bytes >= 128.
- * Generates offset - kMatchBytes of zeros to get a large offset without
- * polluting the hash tables.
- * Generates another kMatchBytes using the saved seed to generate a with the
- * required offset.
- */
-static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
-{
-    typedef enum {
-        of_start = 0,
-        of_run_bytes,
-        of_offset,
-        of_run_match,
-    } of_state;
-    BYTE* const ostart = (BYTE*)out->dst;
-    BYTE* const oend = ostart + out->size;
-    BYTE* op = ostart + out->pos;
-
-    switch ((of_state)stream->state) {
-    case of_start:
-        stream->state = of_run_bytes;
-        stream->saved = stream->seed;
-        stream->bytesLeft = MIN(value, kMatchBytes);
-    /* fall-through */
-    case of_run_bytes: {
-        while (stream->bytesLeft > 0 && op < oend) {
-            *op++ = SEQ_randByte(&stream->seed) | 0x80;
-            --stream->bytesLeft;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-        /* State transition */
-        stream->state = of_offset;
-        stream->bytesLeft = value - MIN(value, kMatchBytes);
-    }
-    /* fall-through */
-    case of_offset: {
-        /* Copy matchLength + 1 bytes to the output buffer */
-        size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
-        if (setLength > 0) {
-            memset(op, 0, setLength);
-            op += setLength;
-            stream->bytesLeft -= setLength;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-        /* State transition */
-        stream->state = of_run_match;
-        stream->bytesLeft = MIN(value, kMatchBytes);
-    }
-    /* fall-through */
-    case of_run_match: {
-        while (stream->bytesLeft > 0 && op < oend) {
-            *op++ = SEQ_randByte(&stream->saved) | 0x80;
-            --stream->bytesLeft;
-        }
-        if (stream->bytesLeft > 0)
-            break;
-    }
-    /* fall-through */
-    default:
-        stream->state = 0;
-        stream->bytesLeft = 0;
-        break;
-    }
-    XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
-    out->pos = op - ostart;
-    return stream->bytesLeft;
-}
-
-/* Returns the number of bytes left to generate.
- * Must pass the same type/value until it returns 0.
- */
-size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out)
-{
-    switch (type) {
-        case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out);
-        case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out);
-        case SEQ_gen_of: return SEQ_gen_offset(stream, value, out);
-        case SEQ_gen_max: /* fall-through */
-        default: return 0;
-    }
-}
-
-/* Returns the xxhash of the data produced so far */
-XXH64_hash_t SEQ_digest(SEQ_stream const* stream)
-{
-    return XXH64_digest(&stream->xxh);
-}

package/src/bsdiff/zstd/tests/seqgen.h

@@ -1,58 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-#ifndef SEQGEN_H
-#define SEQGEN_H
-
-#define XXH_STATIC_LINKING_ONLY
-
-#include "xxhash.h"
-#include <stddef.h>   /* size_t */
-
-typedef enum {
-    SEQ_gen_ml = 0,
-    SEQ_gen_ll,
-    SEQ_gen_of,
-    SEQ_gen_max /* Must be the last value */
-} SEQ_gen_type;
-
-/* Internal state, do not use */
-typedef struct {
-    XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */
-    unsigned seed;
-    int state; /* enum to control state machine (clean=0) */
-    unsigned saved;
-    size_t bytesLeft;
-} SEQ_stream;
-
-SEQ_stream SEQ_initStream(unsigned seed);
-
-typedef struct {
-    void* dst;
-    size_t size;
-    size_t pos;
-} SEQ_outBuffer;
-
-/* Returns non-zero until the current type/value has been generated.
- * Must pass the same type/value until it returns 0.
- *
- * Recommended to pick a value in the middle of the range you want, since there
- * may be some noise that causes actual results to be slightly different.
- * We try to be more accurate for smaller values.
- *
- * NOTE: Very small values don't work well (< 6).
- */
-size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value,
-               SEQ_outBuffer* out);
-
-/* Returns the xxhash of the data produced so far */
-XXH64_hash_t SEQ_digest(SEQ_stream const* stream);
-
-#endif /* SEQGEN_H */

package/src/bsdiff/zstd/tests/test-license.py

@@ -1,156 +0,0 @@
-#!/usr/bin/env python3
-
-# ################################################################
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under both the BSD-style license (found in the
-# LICENSE file in the root directory of this source tree) and the GPLv2 (found
-# in the COPYING file in the root directory of this source tree).
-# You may select, at your option, one of the above-listed licenses.
-# ################################################################
-
-import enum
-import glob
-import os
-import re
-import sys
-
-ROOT = os.path.join(os.path.dirname(__file__), "..")
-
-RELDIRS = [
-    "doc",
-    "examples",
-    "lib",
-    "programs",
-    "tests",
-    "contrib/linux-kernel",
-]
-
-REL_EXCLUDES = [
-    "contrib/linux-kernel/test/include",
-]
-
-def to_abs(d):
-    return os.path.normpath(os.path.join(ROOT, d)) + "/"
-
-DIRS = [to_abs(d) for d in RELDIRS]
-EXCLUDES = [to_abs(d) for d in REL_EXCLUDES]
-
-SUFFIXES = [
-    ".c",
-    ".h",
-    "Makefile",
-    ".mk",
-    ".py",
-    ".S",
-]
-
-# License should certainly be in the first 10 KB.
-MAX_BYTES = 10000
-MAX_LINES = 50
-
-LICENSE_LINES = [
-    "This source code is licensed under both the BSD-style license (found in the",
-    "LICENSE file in the root directory of this source tree) and the GPLv2 (found",
-    "in the COPYING file in the root directory of this source tree).",
-    "You may select, at your option, one of the above-listed licenses.",
-]
-
-COPYRIGHT_EXCEPTIONS = {
-    # From zstdmt
-    "threading.c",
-    "threading.h",
-    # From divsufsort
-    "divsufsort.c",
-    "divsufsort.h",
-}
-
-LICENSE_EXCEPTIONS = {
-    # From divsufsort
-    "divsufsort.c",
-    "divsufsort.h",
-    # License is slightly different because it references GitHub
-    "linux_zstd.h",
-}
-
-
-def valid_copyright(lines):
-    YEAR_REGEX = re.compile("\d\d\d\d|present")
-    for line in lines:
-        line = line.strip()
-        if "Copyright" not in line:
-            continue
-        if "present" in line:
-            return (False, f"Copyright line '{line}' contains 'present'!")
-        if "Meta Platforms, Inc" not in line:
-            return (False, f"Copyright line '{line}' does not contain 'Meta Platforms, Inc'")
-        year = YEAR_REGEX.search(line)
-        if year is not None:
-            return (False, f"Copyright line '{line}' contains {year.group(0)}; it should be yearless")
-        if " (c) " not in line:
-            return (False, f"Copyright line '{line}' does not contain ' (c) '!")
-        return (True, "")
-    return (False, "Copyright not found!")
-
-
-def valid_license(lines):
-    for b in range(len(lines)):
-        if LICENSE_LINES[0] not in lines[b]:
-            continue
-        for l in range(len(LICENSE_LINES)):
-            if LICENSE_LINES[l] not in lines[b + l]:
-                message = f"""Invalid license line found starting on line {b + l}!
-Expected: '{LICENSE_LINES[l]}'
-Actual: '{lines[b + l]}'"""
-                return (False, message)
-        return (True, "")
-    return (False, "License not found!")
-
-
-def valid_file(filename):
-    with open(filename, "r") as f:
-        lines = f.readlines(MAX_BYTES)
-    lines = lines[:min(len(lines), MAX_LINES)]
-
-    ok = True
-    if os.path.basename(filename) not in COPYRIGHT_EXCEPTIONS:
-        c_ok, c_msg = valid_copyright(lines)
-        if not c_ok:
-            print(f"{filename}: {c_msg}", file=sys.stderr)
-            ok = False
-    if os.path.basename(filename) not in LICENSE_EXCEPTIONS:
-        l_ok, l_msg = valid_license(lines)
-        if not l_ok:
-            print(f"{filename}: {l_msg}", file=sys.stderr)
-            ok = False
-    return ok
-
-
-def exclude(filename):
-    for x in EXCLUDES:
-        if filename.startswith(x):
-            return True
-    return False
-
-def main():
-    invalid_files = []
-    for directory in DIRS:
-        for suffix in SUFFIXES:
-            files = set(glob.glob(f"{directory}/**/*{suffix}", recursive=True))
-            for filename in files:
-                if exclude(filename):
-                    continue
-                if not valid_file(filename):
-                    invalid_files.append(filename)
-    if len(invalid_files) > 0:
-        print("Fail!", file=sys.stderr)
-        for f in invalid_files:
-            print(f)
-        return 1
-    else:
-        print("Pass!", file=sys.stderr)
-        return 0
-
-if __name__ == "__main__":
-    sys.exit(main())

package/src/bsdiff/zstd/tests/test-variants.sh

@@ -1,115 +0,0 @@
-#!/bin/sh
-set -e
-set -u
-set -x
-
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-PROG_DIR="$SCRIPT_DIR/../programs"
-
-ZSTD="$PROG_DIR/zstd"
-ZSTD_COMPRESS="$PROG_DIR/zstd-compress"
-ZSTD_DECOMPRESS="$PROG_DIR/zstd-decompress"
-ZSTD_NOLEGACY="$PROG_DIR/zstd-nolegacy"
-ZSTD_DICTBUILDER="$PROG_DIR/zstd-dictBuilder"
-ZSTD_FRUGAL="$PROG_DIR/zstd-frugal"
-ZSTD_NOMT="$PROG_DIR/zstd-nomt"
-
-println() {
-    printf '%b\n' "${*}"
-}
-
-die() {
-    println "$@" 1>&2
-    exit 1
-}
-
-symbol_present() {
-	(nm $1 || echo "symbol_present $@ failed") | grep $2
-}
-
-symbol_not_present() {
-	symbol_present $@ && die "Binary '$1' mistakenly contains symbol '$2'" ||:
-}
-
-compress_not_present() {
-	symbol_not_present "$1" ZSTD_compress
-}
-
-decompress_not_present() {
-	symbol_not_present "$1" ZSTD_decompress
-}
-
-dict_not_present() {
-	symbol_not_present "$1" ZDICT_
-	symbol_not_present "$1" COVER_
-}
-
-cliextra_not_present() {
-	symbol_not_present "$1" TRACE_
-	symbol_not_present "$1" BMK_
-}
-
-legacy_not_present() {
-	symbol_not_present "$1" ZSTDv0
-}
-
-test_help() {
-	"$1" --help | grep -- "$2"
-}
-
-test_no_help() {
-	test_help $@ && die "'$1' supports '$2' when it shouldn't" ||:
-}
-
-extras_not_present() {
-	dict_not_present $@
-	legacy_not_present $@
-	cliextra_not_present $@
-	test_no_help $@ "--train"
-	test_no_help $@ "-b#"
-}
-
-test_compress() {
-	echo "hello" | "$1" | "$ZSTD" -t
-}
-
-test_decompress() {
-	echo "hello" | "$ZSTD" | "$1" -t
-}
-
-test_zstd() {
-	test_compress $@
-	test_decompress $@
-}
-
-extras_not_present "$ZSTD_FRUGAL"
-extras_not_present "$ZSTD_COMPRESS"
-extras_not_present "$ZSTD_DECOMPRESS"
-
-compress_not_present "$ZSTD_DECOMPRESS"
-
-decompress_not_present "$ZSTD_COMPRESS"
-decompress_not_present "$ZSTD_DICTBUILDER"
-
-cliextra_not_present "$ZSTD_DICTBUILDER"
-
-legacy_not_present "$ZSTD_DICTBUILDER"
-legacy_not_present "$ZSTD_NOLEGACY"
-
-symbol_not_present "$ZSTD" ZSTDv01
-symbol_not_present "$ZSTD" ZSTDv02
-symbol_not_present "$ZSTD" ZSTDv03
-symbol_not_present "$ZSTD" ZSTDv04
-
-test_compress "$ZSTD_COMPRESS"
-test_decompress "$ZSTD_DECOMPRESS"
-
-test_zstd "$ZSTD_FRUGAL"
-test_zstd "$ZSTD_NOLEGACY"
-
-test_help "$ZSTD" '-b#'
-test_help "$ZSTD" --train
-test_help "$ZSTD_DICTBUILDER" --train
-
-println "Success!"