zstd-native-ruby 1.0.1 → 1.0.2

data/{vendor → ext/zstd_native}/zstd/programs/zstd.1.md

@@ -113,11 +113,7 @@ the last one takes effect.
     Because the compressor's behavior highly depends on the content to compress, there's no guarantee of a smooth progression from one level to another.
 * `--ultra`:
     unlocks high compression levels 20+ (maximum 22), using a lot more memory.
-    Decompression will also need more memory when using these levels.
-* `--max`:
-    set advanced parameters to reach maximum compression.
-    warning: this setting is very slow and uses a lot of resources.
-    It's inappropriate for 32-bit mode and therefore disabled in this mode.
+    Note that decompression will also require more memory when using these levels.
 * `--fast[=#]`:
     switch to ultra-fast compression levels.
     If `=#` is not present, it defaults to `1`.
@@ -126,7 +122,7 @@ the last one takes effect.
     This setting overwrites compression level if one was set previously.
     Similarly, if a compression level is set after `--fast`, it overrides it.
 * `-T#`, `--threads=#`:
-    Compress using `#` working threads (default: between 1 and 4 depending on physical CPU cores; see `ZSTD_NBTHREADS` below).
+    Compress using `#` working threads (default: 1).
     If `#` is 0, attempt to detect and use the number of physical CPU cores.
     In all cases, the nb of threads is capped to `ZSTDMT_NBWORKERS_MAX`,
     which is either 64 in 32-bit mode, or 256 for 64-bit environments.
@@ -165,6 +161,10 @@ the last one takes effect.
 
     Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
     `--memory=windowSize` needs to be passed to the decompressor.
+* `--max`:
+    set advanced parameters to maximum compression.
+    warning: this setting is very slow and uses a lot of resources.
+    It's inappropriate for 32-bit mode and therefore disabled in this mode.
 * `-D DICT`:
     use `DICT` as Dictionary to compress or decompress FILE(s)
 * `--patch-from FILE`:
@@ -503,12 +503,12 @@ similar to predefined level 19 for files bigger than 256 KB:
 
 `--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
 
-### --jobsize=#:
+### -B#:
 Specify the size of each compression job.
-This parameter is only meaningful when multi-threading is enabled.
-Each compression job is run in parallel, so this value can indirectly impact the nb of active threads.
+This parameter is only available when multi-threading is enabled.
+Each compression job is run in parallel, so this value indirectly impacts the nb of active threads.
 Default job size varies depending on compression level (generally  `4 * windowSize`).
-`--jobsize=#` makes it possible to manually select a custom size.
+`-B#` makes it possible to manually select a custom size.
 Note that job size must respect a minimum value which is enforced transparently.
 This minimum is either 512 KB, or `overlapSize`, whichever is largest.
 Different job sizes will lead to non-identical compressed frames.
@@ -554,8 +554,8 @@ Compression of small files similar to the sample set will be greatly improved.
     Use `#` compression level during training (optional).
     Will generate statistics more tuned for selected compression level,
     resulting in a _small_ compression ratio improvement for this level.
-* `--split=#`:
-    Split input files into independent chunks of size # (default: no split)
+* `-B#`:
+    Split input files into blocks of size # (default: no split)
 * `-M#`, `--memory=#`:
     Limit the amount of sample data loaded for training (default: 2 GB).
     Note that the default (2 GB) is also the maximum.
@@ -683,8 +683,8 @@ Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default i
     benchmark decompression speed only (requires providing a zstd-compressed content)
 * `-i#`:
     minimum evaluation time, in seconds (default: 3s), benchmark mode only
-* `--split=#`:
-    split input file(s) into independent chunks of size # (default: no chunking)
+* `-B#`, `--block-size=#`:
+    cut file(s) into independent chunks of size # (default: no chunking)
 * `-S`:
     output one benchmark result per input file (default: consolidated result)
 * `-D dictionary`

data/{vendor → ext/zstd_native}/zstd/programs/zstdcli.c

@@ -44,10 +44,10 @@
 #endif
 
 #ifndef ZSTDCLI_NBTHREADS_DEFAULT
-#define ZSTDCLI_NBTHREADS_DEFAULT (unsigned)(MAX(1, MIN(4, UTIL_countLogicalCores() / 4)))
+#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4))
 #endif
 
-static unsigned init_nbWorkers(unsigned defaultNbWorkers);
+
 
 /*-************************************
 *  Constants
@@ -94,8 +94,6 @@ static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
 
 
 #define DEFAULT_ACCEL 1
-#define NBWORKERS_AUTOCPU 0
-#define NBWORKERS_UNSET UINT_MAX
 
 typedef enum { cover, fastCover, legacy } dictType;
 
@@ -148,7 +146,7 @@ static void usage(FILE* f, const char* programName)
     DISPLAY_F(f, "Options:\n");
     DISPLAY_F(f, "  -o OUTPUT                     Write output to a single file, OUTPUT.\n");
     DISPLAY_F(f, "  -k, --keep                    Preserve INPUT file(s). [Default] \n");
-    DISPLAY_F(f, "  --rm                          Remove INPUT file(s) after successful (de)compression to file.\n");
+    DISPLAY_F(f, "  --rm                          Remove INPUT file(s) after successful (de)compression.\n");
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
         DISPLAY_F(f, "  -n, --no-name                 Do not store original filename when compressing.\n\n");
@@ -231,15 +229,14 @@ static void usageAdvanced(const char* programName)
 #endif
     DISPLAYOUT("  --adapt                       Dynamically adapt compression level to I/O conditions.\n");
     DISPLAYOUT("  --long[=#]                    Enable long distance matching with window log #. [Default: %u]\n", g_defaultMaxWindowLog);
-    DISPLAYOUT("  --patch-from=REF              Use REF as the reference point for Zstandard's diff engine. \n");
-    DISPLAYOUT("  --patch-apply                 Equivalent for `-d --patch-from` \n\n");
+    DISPLAYOUT("  --patch-from=REF              Use REF as the reference point for Zstandard's diff engine. \n\n");
 # ifdef ZSTD_MULTITHREAD
-    DISPLAYOUT("  -T#                           Spawn # compression threads. [Default: %u; pass 0 for core count.]\n", init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT));
+    DISPLAYOUT("  -T#                           Spawn # compression threads. [Default: 1; pass 0 for core count.]\n");
     DISPLAYOUT("  --single-thread               Share a single thread for I/O and compression (slightly different than `-T1`).\n");
     DISPLAYOUT("  --auto-threads={physical|logical}\n");
     DISPLAYOUT("                                Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
-    DISPLAYOUT("  --jobsize=#                   Set job size to #. [Default: 0 (automatic)]\n");
-    DISPLAYOUT("  --rsyncable                   Compress using a rsync-friendly method (`--jobsize=#` sets unit size). \n");
+    DISPLAYOUT("  -B#                           Set job size to #. [Default: 0 (automatic)]\n");
+    DISPLAYOUT("  --rsyncable                   Compress using a rsync-friendly method (`-B` sets block size). \n");
     DISPLAYOUT("\n");
 # endif
     DISPLAYOUT("  --exclude-compressed          Only compress files that are not already compressed.\n\n");
@@ -310,7 +307,7 @@ static void usageAdvanced(const char* programName)
     DISPLAYOUT("  -b#                           Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
     DISPLAYOUT("  -e#                           Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
     DISPLAYOUT("  -i#                           Set the minimum evaluation to time # seconds. [Default: 3]\n");
-    DISPLAYOUT("  --split=#                     Split input into independent chunks of size #. [Default: No chunking]\n");
+    DISPLAYOUT("  -B#                           Cut file into independent chunks of size #. [Default: No chunking]\n");
     DISPLAYOUT("  -S                            Output one benchmark result per input file. [Default: Consolidated result]\n");
     DISPLAYOUT("  -D dictionary                 Benchmark using dictionary \n");
     DISPLAYOUT("  --priority=rt                 Set process priority to real-time.\n");
@@ -688,12 +685,6 @@ static void printVersion(void)
             DISPLAYOUT("lz4 version %s\n", FIO_lz4Version());
             DISPLAYOUT("lzma version %s\n", FIO_lzmaVersion());
 
-        #ifdef ZSTD_MULTITHREAD
-            DISPLAYOUT("supports Multithreading \n");
-        #else
-            DISPLAYOUT("single-thread operations only \n");
-        #endif
-
             /* posix support */
         #ifdef _POSIX_C_SOURCE
             DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
@@ -704,10 +695,7 @@ static void printVersion(void)
         #ifdef PLATFORM_POSIX_VERSION
             DISPLAYOUT("PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
         #endif
-
-            if (!ZSTD_isDeterministicBuild()) {
-                DISPLAYOUT("non-deterministic build\n");
-    }   }   }
+    }   }
 }
 
 #define ZSTD_NB_STRATEGIES 9
@@ -754,7 +742,7 @@ static void printActualCParams(const char* filename, const char* dictFileName, i
 
 /* Environment variables for parameter setting */
 #define ENV_CLEVEL "ZSTD_CLEVEL"
-#define ENV_NBWORKERS "ZSTD_NBTHREADS"    /* takes lower precedence than directly specifying -T# in the CLI */
+#define ENV_NBTHREADS "ZSTD_NBTHREADS"    /* takes lower precedence than directly specifying -T# in the CLI */
 
 /* pick up environment variable */
 static int init_cLevel(void) {
@@ -784,29 +772,26 @@ static int init_cLevel(void) {
     return ZSTDCLI_CLEVEL_DEFAULT;
 }
 
-static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
 #ifdef ZSTD_MULTITHREAD
-    const char* const env = getenv(ENV_NBWORKERS);
+static unsigned default_nbThreads(void) {
+    const char* const env = getenv(ENV_NBTHREADS);
     if (env != NULL) {
         const char* ptr = env;
         if ((*ptr>='0') && (*ptr<='9')) {
             unsigned nbThreads;
             if (readU32FromCharChecked(&ptr, &nbThreads)) {
-                DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBWORKERS, env);
-                return defaultNbWorkers;
+                DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env);
+                return ZSTDCLI_NBTHREADS_DEFAULT;
             } else if (*ptr == 0) {
                 return nbThreads;
             }
         }
-        DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBWORKERS, env);
+        DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env);
     }
 
-    return defaultNbWorkers;
-#else
-    (void)defaultNbWorkers;
-    return 1;
-#endif
+    return ZSTDCLI_NBTHREADS_DEFAULT;
 }
+#endif
 
 #define NEXT_FIELD(ptr) {         \
     if (*argument == '=') {       \
@@ -825,22 +810,22 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
             CLEAN_RETURN(1);      \
 }   }   }
 
-#define NEXT_UINT32(_varu32) {        \
-    const char* __nb;                 \
-    NEXT_FIELD(__nb);                 \
-    _varu32 = readU32FromChar(&__nb); \
-    if(*__nb != 0) {                  \
+#define NEXT_UINT32(val32) {        \
+    const char* __nb;               \
+    NEXT_FIELD(__nb);               \
+    val32 = readU32FromChar(&__nb); \
+    if(*__nb != 0) {                \
         errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
-    }                                 \
+    }                               \
 }
 
-#define NEXT_TSIZE(_varTsize) {           \
-    const char* __nb;                     \
-    NEXT_FIELD(__nb);                     \
-    _varTsize = readSizeTFromChar(&__nb); \
-    if(*__nb != 0) {                      \
+#define NEXT_TSIZE(valTsize) {           \
+    const char* __nb;                    \
+    NEXT_FIELD(__nb);                    \
+    valTsize = readSizeTFromChar(&__nb); \
+    if(*__nb != 0) {                     \
         errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
-    }                                     \
+    }                                    \
 }
 
 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
@@ -877,25 +862,24 @@ int main(int argCount, const char* argv[])
         singleThread = 0,
         defaultLogicalCores = 0,
         showDefaultCParams = 0,
-        contentSize = 1,
-        removeSrcFile = 0,
-        cLevel = init_cLevel(),
-        ultra = 0,
-        cLevelLast = MINCLEVEL - 1, /* for benchmark range */
-        setThreads_non1 = 0;
-    unsigned nbWorkers = init_nbWorkers(NBWORKERS_UNSET);
-    ZSTD_ParamSwitch_e mmapDict = ZSTD_ps_auto;
+        ultra=0,
+        contentSize=1,
+        removeSrcFile=0;
+    ZSTD_ParamSwitch_e mmapDict=ZSTD_ps_auto;
     ZSTD_ParamSwitch_e useRowMatchFinder = ZSTD_ps_auto;
     FIO_compressionType_t cType = FIO_zstdCompression;
+    int nbWorkers = -1; /* -1 means unset */
     double compressibility = -1.0;  /* lorem ipsum generator */
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
-    size_t chunkSize = 0;
+    size_t blockSize = 0;
 
     FIO_prefs_t* const prefs = FIO_createPreferences();
     FIO_ctx_t* const fCtx = FIO_createContext();
     FIO_progressSetting_e progress = FIO_ps_auto;
     zstd_operation_mode operation = zom_compress;
     ZSTD_compressionParameters compressionParams;
+    int cLevel = init_cLevel();
+    int cLevelLast = MINCLEVEL - 1;  /* lower than minimum */
     unsigned recursive = 0;
     unsigned memLimit = 0;
     FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount);  /* argCount >= 1 */
@@ -934,7 +918,7 @@ int main(int argCount, const char* argv[])
     programName = lastNameFromPath(programName);
 
     /* preset behaviors */
-    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=NBWORKERS_AUTOCPU, singleThread=0;
+    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
     if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
@@ -942,7 +926,7 @@ int main(int argCount, const char* argv[])
         suffix = GZ_EXTENSION; cType = FIO_gzipCompression; removeSrcFile=1;
         dictCLevel = cLevel = 6;  /* gzip default is -6 */
     }
-    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; }                              /* behave like gunzip, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; removeSrcFile=1; }                                                     /* behave like gunzip, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
     if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; cType = FIO_lzmaCompression; removeSrcFile=1; }    /* behave like lzma */
     if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; cType = FIO_lzmaCompression; removeSrcFile=1; } /* behave like unlzma, also supports multiple formats */
@@ -1085,13 +1069,11 @@ int main(int argCount, const char* argv[])
                   continue;
                 }
 #endif
-                if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); setThreads_non1 = (nbWorkers != 1); continue; }
+                if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; }
                 if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
                 if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
                 if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
-                if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(chunkSize); continue; } /* hidden command, prefer --split below */
-                if (longCommandWArg(&argument, "--split")) { NEXT_TSIZE(chunkSize); continue; }
-                if (longCommandWArg(&argument, "--jobsize")) { NEXT_TSIZE(chunkSize); continue; } /* note: overloaded variable */
+                if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
                 if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
                 if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
                 if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
@@ -1127,7 +1109,6 @@ int main(int argCount, const char* argv[])
                 if (longCommandWArg(&argument, "--trace")) { char const* traceFile; NEXT_FIELD(traceFile); TRACE_enable(traceFile); continue; }
 #endif
                 if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); ultra = 1; continue; }
-                if (longCommandWArg(&argument, "--patch-apply")) { operation=zom_decompress; NEXT_FIELD(patchFromDictFileName); memLimit= 1U << ZSTD_WINDOWLOG_MAX; continue; }
                 if (longCommandWArg(&argument, "--long")) {
                     unsigned ldmWindowLog = 0;
                     ldmFlag = 1;
@@ -1275,10 +1256,10 @@ int main(int argCount, const char* argv[])
                     bench_nbSeconds = readU32FromChar(&argument);
                     break;
 
-                    /* hidden shortcut for --split=# and --jobsize=# */
+                    /* cut input into blocks (benchmark only) */
                 case 'B':
                     argument++;
-                    chunkSize = readU32FromChar(&argument);
+                    blockSize = readU32FromChar(&argument);
                     break;
 
                     /* benchmark files separately (hidden option) */
@@ -1293,7 +1274,6 @@ int main(int argCount, const char* argv[])
                 case 'T':
                     argument++;
                     nbWorkers = readU32FromChar(&argument);
-                    setThreads_non1 = (nbWorkers != 1);
                     break;
 
                     /* Dictionary Selection level */
@@ -1338,11 +1318,11 @@ int main(int argCount, const char* argv[])
     DISPLAYLEVEL(3, WELCOME_MESSAGE);
 
 #ifdef ZSTD_MULTITHREAD
-    if ((operation==zom_decompress) && (setThreads_non1)) {
+    if ((operation==zom_decompress) && (nbWorkers > 1)) {
         DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
     }
-    if ((nbWorkers==NBWORKERS_AUTOCPU) && (!singleThread)) {
-        /* automatically set # workers based on # of reported cpu cores */
+    if ((nbWorkers==0) && (!singleThread)) {
+        /* automatically set # workers based on # of reported cpus */
         if (defaultLogicalCores) {
             nbWorkers = (unsigned)UTIL_countLogicalCores();
             DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
@@ -1351,10 +1331,18 @@ int main(int argCount, const char* argv[])
             DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
         }
     }
-    if (operation == zom_compress)
+    /* Resolve to default if nbWorkers is still unset */
+    if (nbWorkers == -1) {
+      if (operation == zom_decompress) {
+        nbWorkers = 1;
+      } else {
+        nbWorkers = default_nbThreads();
+      }
+    }
+    if (operation != zom_bench)
         DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
 #else
-    (void)singleThread; (void)nbWorkers; (void)defaultLogicalCores; (void)setThreads_non1;
+    (void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
 #endif
 
     g_utilDisplayLevel = g_displayLevel;
@@ -1381,7 +1369,7 @@ int main(int argCount, const char* argv[])
         size_t const nbFileLists = file_of_names->tableSize;
         size_t flNb;
         for (flNb=0; flNb < nbFileLists; flNb++) {
-            FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileList(file_of_names->fileNames[flNb]);
+            FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileName(file_of_names->fileNames[flNb]);
             if (fnt==NULL) {
                 DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]);
                 CLEAN_RETURN(1);
@@ -1412,14 +1400,11 @@ int main(int argCount, const char* argv[])
     /* Check if benchmark is selected */
     if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
-        int benchThreadsMsgLevel = 2;
-        if (nbWorkers == NBWORKERS_UNSET) nbWorkers = 1;
-        if (nbWorkers <= 1) benchThreadsMsgLevel = 3;
         if (cType != FIO_zstdCompression) {
             DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
             CLEAN_RETURN(1);
         }
-        benchParams.chunkSizeMax = chunkSize;
+        benchParams.blockSize = blockSize;
         benchParams.targetCBlockSize = targetCBlockSize;
         benchParams.nbWorkers = (int)nbWorkers;
         benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1440,7 +1425,7 @@ int main(int argCount, const char* argv[])
         if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
         if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
         if (cLevelLast < cLevel) cLevelLast = cLevel;
-        DISPLAYLEVEL(benchThreadsMsgLevel, "Benchmarking ");
+        DISPLAYLEVEL(3, "Benchmarking ");
         if (filenames->tableSize > 1)
             DISPLAYLEVEL(3, "%u files ", (unsigned)filenames->tableSize);
         if (cLevelLast > cLevel) {
@@ -1448,7 +1433,7 @@ int main(int argCount, const char* argv[])
         } else {
             DISPLAYLEVEL(3, "at level %d ", cLevel);
         }
-        DISPLAYLEVEL(benchThreadsMsgLevel, "using %i threads \n", nbWorkers);
+        DISPLAYLEVEL(3, "using %i threads \n", nbWorkers);
         if (filenames->tableSize > 0) {
             if(separateFiles) {
                 unsigned i;
@@ -1463,7 +1448,7 @@ int main(int argCount, const char* argv[])
         }
 
 #else
-        (void)bench_nbSeconds; (void)chunkSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
+        (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
 #endif
         goto _end;
     }
@@ -1475,23 +1460,22 @@ int main(int argCount, const char* argv[])
         zParams.compressionLevel = dictCLevel;
         zParams.notificationLevel = (unsigned)g_displayLevel;
         zParams.dictID = dictID;
-        if (nbWorkers == NBWORKERS_UNSET) nbWorkers = init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT);
         if (dict == cover) {
             int const optimize = !coverParams.k || !coverParams.d;
             coverParams.nbThreads = (unsigned)nbWorkers;
             coverParams.zParams = zParams;
-            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, &coverParams, NULL, optimize, memLimit);
+            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize, memLimit);
         } else if (dict == fastCover) {
             int const optimize = !fastCoverParams.k || !fastCoverParams.d;
             fastCoverParams.nbThreads = (unsigned)nbWorkers;
             fastCoverParams.zParams = zParams;
-            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
+            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
         } else {
             ZDICT_legacy_params_t dictParams;
             memset(&dictParams, 0, sizeof(dictParams));
             dictParams.selectivityLevel = dictSelect;
             dictParams.zParams = zParams;
-            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, &dictParams, NULL, NULL, 0, memLimit);
+            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0, memLimit);
         }
 #else
         (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
@@ -1598,9 +1582,8 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOCOMPRESS
         FIO_setCompressionType(prefs, cType);
         FIO_setContentSize(prefs, contentSize);
-        if (nbWorkers == NBWORKERS_UNSET) nbWorkers = init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT);
         FIO_setNbWorkers(prefs, (int)nbWorkers);
-        FIO_setJobSize(prefs, (int)chunkSize);
+        FIO_setBlockSize(prefs, (int)blockSize);
         if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
         FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
         FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);

data/{vendor → ext/zstd_native}/zstd/programs/zstdcli_trace.c

@@ -81,9 +81,8 @@ static void TRACE_log(char const* method, PTime duration, ZSTD_Trace const* trac
 {
     int level = 0;
     int workers = 0;
-    double const durationAsDouble = duration ? (double)duration : .1;
     double const ratio = (double)trace->uncompressedSize / (double)trace->compressedSize;
-    double const speed = ((double)trace->uncompressedSize * 1000) / durationAsDouble;
+    double const speed = ((double)trace->uncompressedSize * 1000) / (double)duration;
     if (trace->params) {
         ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_compressionLevel, &level);
         ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_nbWorkers, &workers);

data/{vendor → ext/zstd_native}/zstd/programs/zstdgrep.1

@@ -1,5 +1,5 @@
 .
-.TH "ZSTDGREP" "1" "February 2025" "zstd 1.5.7" "User Commands"
+.TH "ZSTDGREP" "1" "March 2024" "zstd 1.5.6" "User Commands"
 .
 .SH "NAME"
 \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files

data/{vendor → ext/zstd_native}/zstd/programs/zstdless.1

@@ -1,5 +1,5 @@
 .
-.TH "ZSTDLESS" "1" "February 2025" "zstd 1.5.7" "User Commands"
+.TH "ZSTDLESS" "1" "March 2024" "zstd 1.5.6" "User Commands"
 .
 .SH "NAME"
 \fBzstdless\fR \- view zstandard\-compressed files

data/{vendor → ext/zstd_native}/zstd/tests/Makefile

@@ -359,11 +359,7 @@ test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen
 test-cli-tests: ZSTD = $(PRGDIR)/zstd
 test-cli-tests: zstd datagen
 	file $(ZSTD)
-	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen $(CLI_TEST_ARGS)
-
-.PHONY: update-cli-tests
-update-cli-tests: CLI_TEST_ARGS = --set-exact-output
-update-cli-tests: test-cli-tests
+	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
 
 .PHONY: test-fullbench
 test-fullbench: fullbench datagen
@@ -447,9 +443,9 @@ test-decodecorpus-cli: decodecorpus
 test-pool: poolTests
 	$(QEMU_SYS) ./poolTests
 
-test-lz4: ZSTD = $(PRGDIR)/zstd
-test-lz4: ZSTD_LZ4 = ./lz4
-test-lz4: ZSTD_UNLZ4 = ./unlz4
+test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
+test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
+test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
 test-lz4: zstd decodecorpus datagen
 	[ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
 	[ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4

data/{vendor → ext/zstd_native}/zstd/tests/automated_benchmarking.py

@@ -225,7 +225,7 @@ def get_regressions_dictionary(baseline_build, test_build, filenames_directory,
     for j, level in enumerate(levels):
         old_cspeed, old_dspeed = old[j]
         new_cspeed, new_dspeed = new[j]
-        cspeed_reg, dspeed_reg, baseline_label, test_label = parse_regressions_and_labels(
+        cspeed_reg, dspeed_reg, baesline_label, test_label = parse_regressions_and_labels(
             old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
         )
         if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
@@ -315,7 +315,7 @@ if __name__ == "__main__":
         quit()
 
     if mode == "onetime":
-        main(filenames, levels, iterations, frequency=frequency, dictionary_filename=dictionary_filename)
+        main(filenames, levels, iterations, frequency=frequenc, dictionary_filename=dictionary_filename)
     elif mode == "current":
         builds = [{"user": None, "branch": "None", "hash": None}]
         main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)

data/{vendor/zstd/tests/largeDictionary.c → ext/zstd_native/zstd/tests/bigdict.c}

@@ -70,38 +70,39 @@ int main(int argc, const char** argv)
     char* buffer = (char*)malloc(bufferSize);
     void* out = malloc(outSize);
     void* roundtrip = malloc(dataSize);
-    int _exit_code = 1;
+    int _exit_code = 0;
     (void)argc;
     (void)argv;
 
     if (!buffer || !out || !roundtrip || !cctx || !dctx) {
         fprintf(stderr, "Allocation failure\n");
+        _exit_code = 1;
         goto cleanup;
     }
 
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10)))
-        goto cleanup;
+        return 1;
     if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10)))
-        goto cleanup;
+        return 1;
 
     if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31)))
-        goto cleanup;
+        return 1;
 
     RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe);
 
@@ -111,21 +112,22 @@ int main(int argc, const char** argv)
         for (i = 0; i < 10; ++i) {
             fprintf(stderr, "Compressing 1 GB\n");
             if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue))
-                goto cleanup;
+                return 1;
         }
     }
     fprintf(stderr, "Compressing 1 GB\n");
     if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end))
-        goto cleanup;
+        return 1;
 
-    _exit_code = 0;
     fprintf(stderr, "Success!\n");
 
+    goto cleanup;
+
 cleanup:
     free(roundtrip);
     free(out);
     free(buffer);
-    ZSTD_freeDCtx(dctx);
     ZSTD_freeCCtx(cctx);
+    ZSTD_freeDCtx(dctx);
     return _exit_code;
 }

data/{vendor → ext/zstd_native}/zstd/tests/cli-tests/basic/help.sh.stdout.glob

@@ -6,7 +6,7 @@ Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
 Options:
   -o OUTPUT                     Write output to a single file, OUTPUT.
   -k, --keep                    Preserve INPUT file(s). *Default*
-  --rm                          Remove INPUT file(s) after successful (de)compression to file.
+  --rm                          Remove INPUT file(s) after successful (de)compression.
 
   -#                            Desired compression level, where `#` is a number between 1 and 19;
                                 lower numbers provide faster compression, higher numbers yield

data/{vendor → ext/zstd_native}/zstd/tests/cli-tests/common/platform.sh

@@ -23,11 +23,6 @@ case "$UNAME" in
   *) MD5SUM="md5sum" ;;
 esac
 
-md5hash() {
-  $MD5SUM | dd bs=1 count=32 status=none
-  echo
-}
-
 DIFF="diff"
 case "$UNAME" in
   SunOS) DIFF="gdiff" ;;
@@ -39,9 +34,3 @@ then
 else
     hasMT="true"
 fi
-
-if zstd -vv --version | grep -q 'non-deterministic'; then
-  NON_DETERMINISTIC="true"
-else
-  NON_DETERMINISTIC=""
-fi