zstd-native-ruby 1.0.0

data/vendor/zstd/build/single_file_libs/combine.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+
+# Tool to bundle multiple C/C++ source files, inlining any includes.
+# 
+# Note: there are two types of exclusion options: the '-x' flag, which besides
+# excluding a file also adds an #error directive in place of the #include, and
+# the '-k' flag, which keeps the #include and doesn't inline the file. The
+# intended use cases are: '-x' for files that would normally be #if'd out, so
+# features that 100% won't be used in the amalgamated file, for which every
+# occurrence adds the error, and '-k' for headers that we wish to manually
+# include, such as a project's public API, for which occurrences after the first
+# are removed.
+# 
+# Todo: the error handling could be better, which currently throws and halts
+# (which is functional just not very friendly).
+# 
+# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)
+
+import argparse, re, sys
+
+from pathlib import Path
+from typing import Any, List, Optional, Pattern, Set, TextIO
+
+# Set of file roots when searching (equivalent to -I paths for the compiler).
+roots: Set[Path] = set()
+
+# Set of (canonical) file Path objects to exclude from inlining (and not only
+# exclude but to add a compiler error directive when they're encountered).
+excludes: Set[Path] = set()
+
+# Set of (canonical) file Path objects to keep as include directives.
+keeps: Set[Path] = set()
+
+# Whether to keep the #pragma once directives (unlikely, since this will result
+# in a warning, but the option is there).
+keep_pragma: bool = False
+
+# Destination file object (or stdout if no output file was supplied).
+destn: TextIO = sys.stdout
+
+# Set of file Path objects previously inlined (and to ignore if reencountering).
+found: Set[Path] = set()
+
+# Compiled regex Pattern to handle "#pragma once" in various formats:
+# 
+#   #pragma once
+#     #pragma once
+#   #  pragma once
+#   #pragma   once
+#   #pragma once // comment
+# 
+# Ignoring commented versions, same as include_regex.
+# 
+pragma_regex: Pattern = re.compile(r'^\s*#\s*pragma\s*once\s*')
+
+# Compiled regex Pattern to handle the following type of file includes:
+# 
+#   #include "file"
+#     #include "file"
+#   #  include "file"
+#   #include   "file"
+#   #include "file" // comment
+#   #include "file" // comment with quote "
+# 
+# And all combinations of, as well as ignoring the following:
+# 
+#   #include <file>
+#   //#include "file"
+#   /*#include "file"*/
+# 
+# We don't try to catch errors since the compiler will do this (and the code is
+# expected to be valid before processing) and we don't care what follows the
+# file (whether it's a valid comment or not, since anything after the quoted
+# string is ignored)
+# 
+include_regex: Pattern = re.compile(r'^\s*#\s*include\s*"(.+?)"')
+
+# Simple tests to prove include_regex's cases.
+# 
+def test_match_include() -> bool:
+    if (include_regex.match('#include "file"')   and
+        include_regex.match('  #include "file"') and
+        include_regex.match('#  include "file"') and
+        include_regex.match('#include   "file"') and
+        include_regex.match('#include "file" // comment')):
+            if (not include_regex.match('#include <file>')   and
+                not include_regex.match('//#include "file"') and
+                not include_regex.match('/*#include "file"*/')):
+                    found = include_regex.match('#include "file" // "')
+                    if (found and found.group(1) == 'file'):
+                        print('#include match valid')
+                        return True
+    return False
+
+# Simple tests to prove pragma_regex's cases.
+# 
+def test_match_pragma() -> bool:
+    if (pragma_regex.match('#pragma once')   and
+        pragma_regex.match('  #pragma once') and
+        pragma_regex.match('#  pragma once') and
+        pragma_regex.match('#pragma   once') and
+        pragma_regex.match('#pragma once // comment')):
+            if (not pragma_regex.match('//#pragma once') and
+                not pragma_regex.match('/*#pragma once*/')):
+                    print('#pragma once match valid')
+                    return True
+    return False
+
+# Finds 'file'. First the list of 'root' paths are searched, followed by the
+# currently processing file's 'parent' path, returning a valid Path in
+# canonical form. If no match is found None is returned.
+# 
+def resolve_include(file: str, parent: Optional[Path] = None) -> Optional[Path]:
+    for root in roots:
+        found = root.joinpath(file).resolve()
+        if (found.is_file()):
+            return found
+    if (parent):
+        found = parent.joinpath(file).resolve();
+    else:
+        found = Path(file)
+    if (found.is_file()):
+        return found
+    return None
+
+# Helper to resolve lists of files. 'file_list' is passed in from the arguments
+# and each entry resolved to its canonical path (like any include entry, either
+# from the list of root paths or the owning file's 'parent', which in this case
+# is case is the input file). The results are stored in 'resolved'.
+# 
+def resolve_excluded_files(file_list: Optional[List[str]], resolved: Set[Path], parent: Optional[Path] = None) -> None:
+    if (file_list):
+        for filename in file_list:
+            found = resolve_include(filename, parent)
+            if (found):
+                resolved.add(found)
+            else:
+                error_line(f'Warning: excluded file not found: {filename}')
+
+# Writes 'line' to the open 'destn' (or stdout).
+# 
+def write_line(line: str) -> None:
+    print(line, file=destn)
+
+# Logs 'line' to stderr. This is also used for general notifications that we
+# don't want to go to stdout (so the source can be piped).
+# 
+def error_line(line: Any) -> None:
+    print(line, file=sys.stderr)
+
+# Inline the contents of 'file' (with any of its includes also inlined, etc.).
+# 
+# Note: text encoding errors are ignored and replaced with ? when reading the
+# input files. This isn't ideal, but it's more than likely in the comments than
+# code and a) the text editor has probably also failed to read the same content,
+# and b) the compiler probably did too.
+# 
+def add_file(file: Path, file_name: str = None) -> None:
+    if (file.is_file()):
+        if (not file_name):
+            file_name = file.name
+        error_line(f'Processing: {file_name}')
+        with file.open('r', errors='replace') as opened:
+            for line in opened:
+                line = line.rstrip('\n')
+                match_include = include_regex.match(line);
+                if (match_include):
+                    # We have a quoted include directive so grab the file
+                    inc_name = match_include.group(1)
+                    resolved = resolve_include(inc_name, file.parent)
+                    if (resolved):
+                        if (resolved in excludes):
+                            # The file was excluded so error if the compiler uses it
+                            write_line(f'#error Using excluded file: {inc_name} (re-amalgamate source to fix)')
+                            error_line(f'Excluding: {inc_name}')
+                        else:
+                            if (resolved not in found):
+                                # The file was not previously encountered
+                                found.add(resolved)
+                                if (resolved in keeps):
+                                    # But the include was flagged to keep as included
+                                    write_line(f'/**** *NOT* inlining {inc_name} ****/')
+                                    write_line(line)
+                                    error_line(f'Not inlining: {inc_name}')
+                                else:
+                                    # The file was neither excluded nor seen before so inline it
+                                    write_line(f'/**** start inlining {inc_name} ****/')
+                                    add_file(resolved, inc_name)
+                                    write_line(f'/**** ended inlining {inc_name} ****/')
+                            else:
+                                write_line(f'/**** skipping file: {inc_name} ****/')
+                    else:
+                        # The include file didn't resolve to a file
+                        write_line(f'#error Unable to find: {inc_name}')
+                        error_line(f'Error: Unable to find: {inc_name}')
+                else:
+                    # Skip any 'pragma once' directives, otherwise write the source line
+                    if (keep_pragma or not pragma_regex.match(line)):
+                        write_line(line)
+    else:
+        error_line(f'Error: Invalid file: {file}')
+
+# Start here
+parser = argparse.ArgumentParser(description='Amalgamate Tool', epilog=f'example: {sys.argv[0]} -r ../my/path -r ../other/path -o out.c in.c')
+parser.add_argument('-r', '--root', action='append', type=Path, help='file root search path')
+parser.add_argument('-x', '--exclude',  action='append', help='file to completely exclude from inlining')
+parser.add_argument('-k', '--keep', action='append', help='file to exclude from inlining but keep the include directive')
+parser.add_argument('-p', '--pragma', action='store_true', default=False, help='keep any "#pragma once" directives (removed by default)')
+parser.add_argument('-o', '--output', type=argparse.FileType('w'), help='output file (otherwise stdout)')
+parser.add_argument('input', type=Path, help='input file')
+args = parser.parse_args()
+
+# Fail early on an invalid input (and store it so we don't recurse)
+args.input = args.input.resolve(strict=True)
+found.add(args.input)
+
+# Resolve all of the root paths upfront (we'll halt here on invalid roots)
+if (args.root):
+    for path in args.root:
+        roots.add(path.resolve(strict=True))
+
+# The remaining params: so resolve the excluded files and #pragma once directive
+resolve_excluded_files(args.exclude, excludes, args.input.parent)
+resolve_excluded_files(args.keep,    keeps,    args.input.parent)
+keep_pragma = args.pragma;
+
+# Then recursively process the input file
+try:
+    if (args.output):
+        destn = args.output
+    add_file(args.input)
+finally:
+    if (destn):
+        destn.close()

data/vendor/zstd/build/single_file_libs/combine.sh

@@ -0,0 +1,249 @@
+#!/bin/sh -e
+
+# Tool to bundle multiple C/C++ source files, inlining any includes.
+# 
+# TODO: ROOTS, FOUND, etc., as arrays (since they fail on paths with spaces)
+# 
+# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)
+
+# Common file roots
+ROOTS="."
+
+# -x option excluded includes
+XINCS=""
+
+# -k option includes to keep as include directives
+KINCS=""
+
+# Files previously visited
+FOUND=""
+
+# Optional destination file (empty string to write to stdout)
+DESTN=""
+
+# Whether the "#pragma once" directives should be written to the output
+PONCE=0
+
+# Prints the script usage then exits
+usage() {
+  echo "Usage: $0 [-r <path>] [-x <header>] [-k <header>] [-o <outfile>] infile"
+  echo "  -r file root search path"
+  echo "  -x file to completely exclude from inlining"
+  echo "  -k file to exclude from inlining but keep the include directive"
+  echo "  -p keep any '#pragma once' directives (removed by default)"
+  echo "  -o output file (otherwise stdout)"
+  echo "Example: $0 -r ../my/path - r ../other/path -o out.c in.c"
+  exit 1
+}
+
+# Tests that the grep implementation works as expected (older OSX grep fails)
+test_deps() {
+  if ! echo '#include "foo"' | grep -Eq '^\s*#\s*include\s*".+"'; then
+    echo "Aborting: the grep implementation fails to parse include lines"
+    exit 1
+  fi
+  if ! echo '"foo.h"' | sed -E 's/"([^"]+)"/\1/' | grep -Eq '^foo\.h$'; then
+    echo "Aborting: sed is unavailable or non-functional"
+    exit 1
+  fi
+}
+
+# Test if glob pattern $1 matches subject $2 (see fnmatch(3))
+fnmatch() {
+  case "$2" in
+  $1)
+    return 0
+    ;;
+  esac
+  return 1
+}
+
+# Test if line $1 is local include directive
+is_include_line() {
+  fnmatch "*#*include*" "$1" || return 1
+  printf "%s\n" "$1" | grep -Eq '^\s*#\s*include\s*".+"'
+}
+
+# Test if line $1 is pragma once directive
+is_pragma_once_line() {
+  fnmatch "*#*pragma*once*" "$1" || return 1
+  printf "%s\n" "$1" | grep -Eq '^\s*#\s*pragma\s*once\s*'
+}
+
+# Tests if list $1 has item $2 (returning zero on a match)
+# (originally used grep -Eq "(^|\s*)$2(\$|\s*))
+readonly list_FS="$IFS"
+list_has_item() {
+  # Re: escaping glob pattern special characters in item string:
+  #
+  # bash (tested 3.2.57, 5.1.4), dash (tested 0.5.10.2), NetBSD /bin/sh
+  # (tested 8.2), and Solaris /bin/sh (tested 11.4) require escaping
+  # backslashes in a bracket expression despite POSIX specifying that
+  # backslash loses significance in a bracket expression.
+  #
+  # Conversely, neither FreeBSD /bin/sh (tested 12.2) nor OpenBSD /bin/sh
+  # (tested 7.1) obey backslash-escaping in case statement patterns even
+  # outside bracket expressions, so escape special characters using bracket
+  # expressions.
+  #
+  # Solaris /bin/sh (tested 11.4) requires vertical bar (|) to be escaped.
+  #
+  # All accommodations should behave as expected under strict POSIX semantics.
+  if fnmatch "*[\\*?[|]*" "$2"; then
+    set -- "$1" "$(printf '%s\n' "$2" | sed -e 's/[*?[|]/[&]/g; s/[\]/[\\&]/g')"
+  fi
+  for item_P in "*[$list_FS]$2[$list_FS]*" "*[$list_FS]$2" "$2[$list_FS]*" "$2"; do
+    fnmatch "${item_P}" "$1" && return 0
+  done
+  return 1
+}
+
+# Adds a new line with the supplied arguments to $DESTN (or stdout)
+write_line() {
+  if [ -n "$DESTN" ]; then
+    printf '%s\n' "$@" >> "$DESTN"
+  else
+    printf '%s\n' "$@"
+  fi
+}
+
+log_line() {
+  echo $@ >&2
+}
+
+# Find this file!
+resolve_include() {
+  local srcdir=$1
+  local inc=$2
+  for root in $srcdir $ROOTS; do
+    if [ -f "$root/$inc" ]; then
+      # Try to reduce the file path into a canonical form (so that multiple)
+      # includes of the same file are successfully deduplicated, even if they
+      # are expressed differently.
+      local relpath="$(realpath --relative-to . "$root/$inc" 2>/dev/null)"
+      if [ "$relpath" != "" ]; then # not all realpaths support --relative-to
+        echo "$relpath"
+        return 0
+      fi
+      local relpath="$(realpath "$root/$inc" 2>/dev/null)"
+      if [ "$relpath" != "" ]; then # not all distros have realpath...
+        echo "$relpath"
+        return 0
+      fi
+      # Fallback on Python to reduce the path if the above fails.
+      local relpath=$(python -c "import os,sys; print os.path.relpath(sys.argv[1])" "$root/$inc" 2>/dev/null)
+      if [ "$relpath" != "" ]; then # not all distros have realpath...
+        echo "$relpath"
+        return 0
+      fi
+      # Worst case, fall back to just the root + relative include path. The
+      # problem with this is that it is possible to emit multiple different
+      # resolved paths to the same file, depending on exactly how its included.
+      # Since the main loop below keeps a list of the resolved paths it's
+      # already included, in order to avoid repeated includes, this failure to
+      # produce a canonical/reduced path can lead to multiple inclusions of the
+      # same file. But it seems like the resulting single file library still
+      # works (hurray include guards!), so I guess it's ok.
+      echo "$root/$inc"
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Adds the contents of $1 with any of its includes inlined
+add_file() {
+  local file=$1
+  if [ -n "$file" ]; then
+    log_line "Processing: $file"
+    # Get directory of the current so we can resolve relative includes
+    local srcdir="$(dirname "$file")"
+    # Read the file
+    local line=
+    while IFS= read -r line; do
+      if is_include_line "$line"; then
+        # We have an include directive so strip the (first) file
+        local inc=$(echo "$line" | grep -Eo '".*"' | sed -E 's/"([^"]+)"/\1/' | head -1)
+        local res_inc="$(resolve_include "$srcdir" "$inc")"
+        if list_has_item "$XINCS" "$inc"; then
+          # The file was excluded so error if the source attempts to use it
+          write_line "#error Using excluded file: $inc (re-amalgamate source to fix)"
+          log_line "Excluding: $inc"
+        else
+          if ! list_has_item "$FOUND" "$res_inc"; then
+            # The file was not previously encountered
+            FOUND="$FOUND $res_inc"
+            if list_has_item "$KINCS" "$inc"; then
+              # But the include was flagged to keep as included
+              write_line "/**** *NOT* inlining $inc ****/"
+              write_line "$line"
+              log_line "Not Inlining: $inc"
+            else
+              # The file was neither excluded nor seen before so inline it
+              write_line "/**** start inlining $inc ****/"
+              add_file "$res_inc"
+              write_line "/**** ended inlining $inc ****/"
+            fi
+          else
+            write_line "/**** skipping file: $inc ****/"
+          fi
+        fi
+      else
+        # Skip any 'pragma once' directives, otherwise write the source line
+        local write=$PONCE
+        if [ $write -eq 0 ]; then
+          if ! is_pragma_once_line "$line"; then
+            write=1
+          fi
+        fi
+        if [ $write -ne 0 ]; then
+          write_line "$line"
+        fi
+      fi
+    done < "$file"
+  else
+    write_line "#error Unable to find \"$1\""
+    log_line "Error: Unable to find: \"$1\""
+  fi
+}
+
+while getopts ":r:x:k:po:" opts; do
+  case $opts in
+  r)
+    ROOTS="$ROOTS $OPTARG"
+    ;;
+  x)
+    XINCS="$XINCS $OPTARG"
+    ;;
+  k)
+    KINCS="$KINCS $OPTARG"
+    ;;
+  p)
+    PONCE=1
+    ;;
+  o)
+    DESTN="$OPTARG"
+    ;;
+  *)
+    usage
+    ;;
+  esac
+done
+shift $((OPTIND-1))
+
+if [ -n "$1" ]; then
+  if [ -f "$1" ]; then
+    if [ -n "$DESTN" ]; then
+      printf "" > "$DESTN"
+    fi
+    test_deps
+    log_line "Processing using the slower shell script; this might take a while"
+    add_file "$1"
+  else
+    echo "Input file not found: \"$1\""
+    exit 1
+  fi
+else
+  usage
+fi
+exit 0

data/vendor/zstd/build/single_file_libs/create_single_file_decoder.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Where to find the sources
+ZSTD_SRC_ROOT="../../lib"
+
+# Amalgamate the sources
+echo "Amalgamating files..."
+# Using the faster Python script if we have 3.8 or higher
+if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
+  ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
+else
+  ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
+fi
+# Did combining work?
+if [ $? -ne 0 ]; then
+  echo "Combine script: FAILED"
+  exit 1
+fi
+echo "Combine script: PASSED"

data/vendor/zstd/build/single_file_libs/create_single_file_library.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Where to find the sources
+ZSTD_SRC_ROOT="../../lib"
+
+# Amalgamate the sources
+echo "Amalgamating files..."
+# Using the faster Python script if we have 3.8 or higher
+if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
+  ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
+else
+  ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
+fi
+# Did combining work?
+if [ $? -ne 0 ]; then
+  echo "Combine script: FAILED"
+  exit 1
+fi
+echo "Combine script: PASSED"

data/vendor/zstd/build/single_file_libs/examples/README.md

@@ -0,0 +1,11 @@
+# Single File ZStandard Examples
+
+The examples `#include` the generated `zstddeclib.c` directly but work equally as well when including `zstd.h` and compiling the amalgamated source separately.
+
+`simple.c` is the most basic example of decompressing content and verifying the result.
+
+`emscripten.c` is a bare-bones [Emscripten](https://github.com/emscripten-core/emscripten) compiled WebGL demo using Zstd to further compress a DXT1 texture (see the [original PNG image](testcard.png)). The 256x256 texture would normally be 32kB, but even when bundled with the Zstd decompressor the resulting WebAssembly weighs in at 41kB (`shell.html` is a support file to run the Wasm).
+
+`roundtrip.c` is an example to use with the optional [amalgamated library](../create_single_file_library.sh) showing compression the decompression.
+
+The example files in this directory are released under a [Creative Commons Zero license](https://creativecommons.org/publicdomain/zero/1.0/) (or Public Domain, whichever is applicable in your jurisdiction).