zstd-native-ruby 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (651) hide show
  1. checksums.yaml +7 -0
  2. data/CLAUDE.md +101 -0
  3. data/LICENSE +19 -0
  4. data/README.md +47 -0
  5. data/ext/zstd_ruby/extconf.rb +35 -0
  6. data/ext/zstd_ruby/zstd_ruby.c +221 -0
  7. data/lib/zstd_ruby.rb +68 -0
  8. data/vendor/zstd/CHANGELOG +863 -0
  9. data/vendor/zstd/CMakeLists.txt +11 -0
  10. data/vendor/zstd/CODE_OF_CONDUCT.md +5 -0
  11. data/vendor/zstd/CONTRIBUTING.md +494 -0
  12. data/vendor/zstd/COPYING +339 -0
  13. data/vendor/zstd/LICENSE +30 -0
  14. data/vendor/zstd/Makefile +470 -0
  15. data/vendor/zstd/Package.swift +36 -0
  16. data/vendor/zstd/README.md +244 -0
  17. data/vendor/zstd/SECURITY.md +15 -0
  18. data/vendor/zstd/TESTING.md +43 -0
  19. data/vendor/zstd/build/LICENSE +0 -0
  20. data/vendor/zstd/build/README.md +56 -0
  21. data/vendor/zstd/build/VS2008/fullbench/fullbench.vcproj +549 -0
  22. data/vendor/zstd/build/VS2008/fuzzer/fuzzer.vcproj +585 -0
  23. data/vendor/zstd/build/VS2008/zstd/zstd.vcproj +677 -0
  24. data/vendor/zstd/build/VS2008/zstd.sln +56 -0
  25. data/vendor/zstd/build/VS2008/zstdlib/zstdlib.vcproj +635 -0
  26. data/vendor/zstd/build/VS2010/CompileAsCpp.props +8 -0
  27. data/vendor/zstd/build/VS2010/datagen/datagen.vcxproj +170 -0
  28. data/vendor/zstd/build/VS2010/fullbench/fullbench.vcxproj +220 -0
  29. data/vendor/zstd/build/VS2010/fuzzer/fuzzer.vcxproj +224 -0
  30. data/vendor/zstd/build/VS2010/libzstd/libzstd.vcxproj +243 -0
  31. data/vendor/zstd/build/VS2010/libzstd-dll/libzstd-dll.rc +51 -0
  32. data/vendor/zstd/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +250 -0
  33. data/vendor/zstd/build/VS2010/zstd/zstd.rc +51 -0
  34. data/vendor/zstd/build/VS2010/zstd/zstd.vcxproj +266 -0
  35. data/vendor/zstd/build/VS2010/zstd.sln +84 -0
  36. data/vendor/zstd/build/VS_scripts/README.md +64 -0
  37. data/vendor/zstd/build/VS_scripts/build.VS2010.cmd +7 -0
  38. data/vendor/zstd/build/VS_scripts/build.VS2012.cmd +6 -0
  39. data/vendor/zstd/build/VS_scripts/build.VS2013.cmd +7 -0
  40. data/vendor/zstd/build/VS_scripts/build.VS2015.cmd +7 -0
  41. data/vendor/zstd/build/VS_scripts/build.VS2017.cmd +7 -0
  42. data/vendor/zstd/build/VS_scripts/build.VS2017Community.cmd +7 -0
  43. data/vendor/zstd/build/VS_scripts/build.VS2017Enterprise.cmd +7 -0
  44. data/vendor/zstd/build/VS_scripts/build.VS2017Professional.cmd +7 -0
  45. data/vendor/zstd/build/VS_scripts/build.VSPreview.cmd +7 -0
  46. data/vendor/zstd/build/VS_scripts/build.generic.cmd +67 -0
  47. data/vendor/zstd/build/cmake/CMakeLists.txt +81 -0
  48. data/vendor/zstd/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake +138 -0
  49. data/vendor/zstd/build/cmake/CMakeModules/FindLibLZ4.cmake +49 -0
  50. data/vendor/zstd/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake +10 -0
  51. data/vendor/zstd/build/cmake/CMakeModules/JoinPaths.cmake +23 -0
  52. data/vendor/zstd/build/cmake/CMakeModules/ZstdBuild.cmake +42 -0
  53. data/vendor/zstd/build/cmake/CMakeModules/ZstdDependencies.cmake +30 -0
  54. data/vendor/zstd/build/cmake/CMakeModules/ZstdOptions.cmake +68 -0
  55. data/vendor/zstd/build/cmake/CMakeModules/ZstdPackage.cmake +42 -0
  56. data/vendor/zstd/build/cmake/CMakeModules/ZstdVersion.cmake +31 -0
  57. data/vendor/zstd/build/cmake/README.md +155 -0
  58. data/vendor/zstd/build/cmake/contrib/CMakeLists.txt +13 -0
  59. data/vendor/zstd/build/cmake/contrib/gen_html/CMakeLists.txt +30 -0
  60. data/vendor/zstd/build/cmake/contrib/pzstd/CMakeLists.txt +39 -0
  61. data/vendor/zstd/build/cmake/lib/CMakeLists.txt +298 -0
  62. data/vendor/zstd/build/cmake/lib/cmake_uninstall.cmake.in +22 -0
  63. data/vendor/zstd/build/cmake/programs/CMakeLists.txt +144 -0
  64. data/vendor/zstd/build/cmake/tests/CMakeLists.txt +118 -0
  65. data/vendor/zstd/build/cmake/zstdConfig.cmake.in +10 -0
  66. data/vendor/zstd/build/meson/GetZstdLibraryVersion.py +39 -0
  67. data/vendor/zstd/build/meson/InstallSymlink.py +55 -0
  68. data/vendor/zstd/build/meson/README.md +38 -0
  69. data/vendor/zstd/build/meson/contrib/gen_html/meson.build +30 -0
  70. data/vendor/zstd/build/meson/contrib/meson.build +12 -0
  71. data/vendor/zstd/build/meson/contrib/pzstd/meson.build +25 -0
  72. data/vendor/zstd/build/meson/lib/meson.build +175 -0
  73. data/vendor/zstd/build/meson/meson.build +155 -0
  74. data/vendor/zstd/build/meson/meson_options.txt +36 -0
  75. data/vendor/zstd/build/meson/programs/meson.build +124 -0
  76. data/vendor/zstd/build/meson/tests/meson.build +217 -0
  77. data/vendor/zstd/build/meson/tests/valgrindTest.py +90 -0
  78. data/vendor/zstd/build/single_file_libs/README.md +33 -0
  79. data/vendor/zstd/build/single_file_libs/build_decoder_test.sh +91 -0
  80. data/vendor/zstd/build/single_file_libs/build_library_test.sh +98 -0
  81. data/vendor/zstd/build/single_file_libs/combine.py +234 -0
  82. data/vendor/zstd/build/single_file_libs/combine.sh +249 -0
  83. data/vendor/zstd/build/single_file_libs/create_single_file_decoder.sh +19 -0
  84. data/vendor/zstd/build/single_file_libs/create_single_file_library.sh +19 -0
  85. data/vendor/zstd/build/single_file_libs/examples/README.md +11 -0
  86. data/vendor/zstd/build/single_file_libs/examples/emscripten.c +340 -0
  87. data/vendor/zstd/build/single_file_libs/examples/roundtrip.c +83 -0
  88. data/vendor/zstd/build/single_file_libs/examples/shell.html +31 -0
  89. data/vendor/zstd/build/single_file_libs/examples/simple.c +75 -0
  90. data/vendor/zstd/build/single_file_libs/examples/testcard-dxt1.inl +2731 -0
  91. data/vendor/zstd/build/single_file_libs/examples/testcard-zstd.inl +261 -0
  92. data/vendor/zstd/build/single_file_libs/examples/testcard.png +0 -0
  93. data/vendor/zstd/build/single_file_libs/zstd-in.c +91 -0
  94. data/vendor/zstd/build/single_file_libs/zstddeclib-in.c +62 -0
  95. data/vendor/zstd/contrib/VS2005/README.md +3 -0
  96. data/vendor/zstd/contrib/VS2005/fullbench/fullbench.vcproj +440 -0
  97. data/vendor/zstd/contrib/VS2005/fuzzer/fuzzer.vcproj +488 -0
  98. data/vendor/zstd/contrib/VS2005/zstd/zstd.vcproj +552 -0
  99. data/vendor/zstd/contrib/VS2005/zstd.sln +55 -0
  100. data/vendor/zstd/contrib/VS2005/zstdlib/zstdlib.vcproj +546 -0
  101. data/vendor/zstd/contrib/cleanTabs +2 -0
  102. data/vendor/zstd/contrib/diagnose_corruption/Makefile +35 -0
  103. data/vendor/zstd/contrib/diagnose_corruption/check_flipped_bits.c +400 -0
  104. data/vendor/zstd/contrib/docker/Dockerfile +20 -0
  105. data/vendor/zstd/contrib/docker/README.md +20 -0
  106. data/vendor/zstd/contrib/externalSequenceProducer/Makefile +40 -0
  107. data/vendor/zstd/contrib/externalSequenceProducer/README.md +14 -0
  108. data/vendor/zstd/contrib/externalSequenceProducer/main.c +108 -0
  109. data/vendor/zstd/contrib/externalSequenceProducer/sequence_producer.c +80 -0
  110. data/vendor/zstd/contrib/externalSequenceProducer/sequence_producer.h +26 -0
  111. data/vendor/zstd/contrib/freestanding_lib/freestanding.py +774 -0
  112. data/vendor/zstd/contrib/gen_html/Makefile +51 -0
  113. data/vendor/zstd/contrib/gen_html/README.md +31 -0
  114. data/vendor/zstd/contrib/gen_html/gen-zstd-manual.sh +9 -0
  115. data/vendor/zstd/contrib/gen_html/gen_html.cpp +225 -0
  116. data/vendor/zstd/contrib/largeNbDicts/Makefile +58 -0
  117. data/vendor/zstd/contrib/largeNbDicts/README.md +33 -0
  118. data/vendor/zstd/contrib/largeNbDicts/largeNbDicts.c +1087 -0
  119. data/vendor/zstd/contrib/linux-kernel/Makefile +108 -0
  120. data/vendor/zstd/contrib/linux-kernel/README.md +14 -0
  121. data/vendor/zstd/contrib/linux-kernel/btrfs-benchmark.sh +104 -0
  122. data/vendor/zstd/contrib/linux-kernel/btrfs-extract-benchmark.sh +99 -0
  123. data/vendor/zstd/contrib/linux-kernel/decompress_sources.h +34 -0
  124. data/vendor/zstd/contrib/linux-kernel/linux.mk +44 -0
  125. data/vendor/zstd/contrib/linux-kernel/linux_zstd.h +691 -0
  126. data/vendor/zstd/contrib/linux-kernel/mem.h +262 -0
  127. data/vendor/zstd/contrib/linux-kernel/squashfs-benchmark.sh +39 -0
  128. data/vendor/zstd/contrib/linux-kernel/test/Makefile +49 -0
  129. data/vendor/zstd/contrib/linux-kernel/test/include/linux/compiler.h +23 -0
  130. data/vendor/zstd/contrib/linux-kernel/test/include/linux/errno.h +15 -0
  131. data/vendor/zstd/contrib/linux-kernel/test/include/linux/kernel.h +19 -0
  132. data/vendor/zstd/contrib/linux-kernel/test/include/linux/limits.h +15 -0
  133. data/vendor/zstd/contrib/linux-kernel/test/include/linux/math64.h +15 -0
  134. data/vendor/zstd/contrib/linux-kernel/test/include/linux/module.h +20 -0
  135. data/vendor/zstd/contrib/linux-kernel/test/include/linux/printk.h +15 -0
  136. data/vendor/zstd/contrib/linux-kernel/test/include/linux/stddef.h +15 -0
  137. data/vendor/zstd/contrib/linux-kernel/test/include/linux/swab.h +16 -0
  138. data/vendor/zstd/contrib/linux-kernel/test/include/linux/types.h +16 -0
  139. data/vendor/zstd/contrib/linux-kernel/test/include/linux/unaligned.h +187 -0
  140. data/vendor/zstd/contrib/linux-kernel/test/include/linux/xxhash.h +745 -0
  141. data/vendor/zstd/contrib/linux-kernel/test/macro-test.sh +44 -0
  142. data/vendor/zstd/contrib/linux-kernel/test/static_test.c +52 -0
  143. data/vendor/zstd/contrib/linux-kernel/test/test.c +229 -0
  144. data/vendor/zstd/contrib/linux-kernel/zstd_common_module.c +29 -0
  145. data/vendor/zstd/contrib/linux-kernel/zstd_compress_module.c +286 -0
  146. data/vendor/zstd/contrib/linux-kernel/zstd_decompress_module.c +141 -0
  147. data/vendor/zstd/contrib/linux-kernel/zstd_deps.h +121 -0
  148. data/vendor/zstd/contrib/match_finders/README.md +42 -0
  149. data/vendor/zstd/contrib/match_finders/zstd_edist.c +558 -0
  150. data/vendor/zstd/contrib/match_finders/zstd_edist.h +70 -0
  151. data/vendor/zstd/contrib/premake/premake4.lua +6 -0
  152. data/vendor/zstd/contrib/premake/zstd.lua +81 -0
  153. data/vendor/zstd/contrib/pzstd/BUCK +72 -0
  154. data/vendor/zstd/contrib/pzstd/ErrorHolder.h +54 -0
  155. data/vendor/zstd/contrib/pzstd/Logging.h +73 -0
  156. data/vendor/zstd/contrib/pzstd/Makefile +265 -0
  157. data/vendor/zstd/contrib/pzstd/Options.cpp +424 -0
  158. data/vendor/zstd/contrib/pzstd/Options.h +71 -0
  159. data/vendor/zstd/contrib/pzstd/Pzstd.cpp +626 -0
  160. data/vendor/zstd/contrib/pzstd/Pzstd.h +153 -0
  161. data/vendor/zstd/contrib/pzstd/README.md +56 -0
  162. data/vendor/zstd/contrib/pzstd/SkippableFrame.cpp +30 -0
  163. data/vendor/zstd/contrib/pzstd/SkippableFrame.h +64 -0
  164. data/vendor/zstd/contrib/pzstd/images/Cspeed.png +0 -0
  165. data/vendor/zstd/contrib/pzstd/images/Dspeed.png +0 -0
  166. data/vendor/zstd/contrib/pzstd/main.cpp +27 -0
  167. data/vendor/zstd/contrib/pzstd/test/BUCK +37 -0
  168. data/vendor/zstd/contrib/pzstd/test/OptionsTest.cpp +536 -0
  169. data/vendor/zstd/contrib/pzstd/test/PzstdTest.cpp +147 -0
  170. data/vendor/zstd/contrib/pzstd/test/RoundTrip.h +86 -0
  171. data/vendor/zstd/contrib/pzstd/test/RoundTripTest.cpp +84 -0
  172. data/vendor/zstd/contrib/pzstd/utils/BUCK +75 -0
  173. data/vendor/zstd/contrib/pzstd/utils/Buffer.h +99 -0
  174. data/vendor/zstd/contrib/pzstd/utils/FileSystem.h +96 -0
  175. data/vendor/zstd/contrib/pzstd/utils/Likely.h +28 -0
  176. data/vendor/zstd/contrib/pzstd/utils/Portability.h +16 -0
  177. data/vendor/zstd/contrib/pzstd/utils/Range.h +133 -0
  178. data/vendor/zstd/contrib/pzstd/utils/ResourcePool.h +96 -0
  179. data/vendor/zstd/contrib/pzstd/utils/ScopeGuard.h +50 -0
  180. data/vendor/zstd/contrib/pzstd/utils/ThreadPool.h +58 -0
  181. data/vendor/zstd/contrib/pzstd/utils/WorkQueue.h +182 -0
  182. data/vendor/zstd/contrib/pzstd/utils/test/BUCK +35 -0
  183. data/vendor/zstd/contrib/pzstd/utils/test/BufferTest.cpp +89 -0
  184. data/vendor/zstd/contrib/pzstd/utils/test/RangeTest.cpp +82 -0
  185. data/vendor/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp +72 -0
  186. data/vendor/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp +28 -0
  187. data/vendor/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp +71 -0
  188. data/vendor/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp +282 -0
  189. data/vendor/zstd/contrib/recovery/Makefile +35 -0
  190. data/vendor/zstd/contrib/recovery/recover_directory.c +152 -0
  191. data/vendor/zstd/contrib/seekable_format/README.md +42 -0
  192. data/vendor/zstd/contrib/seekable_format/examples/Makefile +55 -0
  193. data/vendor/zstd/contrib/seekable_format/examples/parallel_compression.c +254 -0
  194. data/vendor/zstd/contrib/seekable_format/examples/parallel_processing.c +191 -0
  195. data/vendor/zstd/contrib/seekable_format/examples/seekable_compression.c +136 -0
  196. data/vendor/zstd/contrib/seekable_format/examples/seekable_decompression.c +141 -0
  197. data/vendor/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c +147 -0
  198. data/vendor/zstd/contrib/seekable_format/tests/Makefile +58 -0
  199. data/vendor/zstd/contrib/seekable_format/tests/seekable_tests.c +375 -0
  200. data/vendor/zstd/contrib/seekable_format/zstd_seekable.h +226 -0
  201. data/vendor/zstd/contrib/seekable_format/zstd_seekable_compression_format.md +116 -0
  202. data/vendor/zstd/contrib/seekable_format/zstdseek_compress.c +365 -0
  203. data/vendor/zstd/contrib/seekable_format/zstdseek_decompress.c +600 -0
  204. data/vendor/zstd/contrib/seqBench/Makefile +58 -0
  205. data/vendor/zstd/contrib/seqBench/seqBench.c +53 -0
  206. data/vendor/zstd/contrib/snap/snapcraft.yaml +28 -0
  207. data/vendor/zstd/doc/README.md +26 -0
  208. data/vendor/zstd/doc/decompressor_errata.md +148 -0
  209. data/vendor/zstd/doc/decompressor_permissive.md +80 -0
  210. data/vendor/zstd/doc/educational_decoder/Makefile +62 -0
  211. data/vendor/zstd/doc/educational_decoder/README.md +36 -0
  212. data/vendor/zstd/doc/educational_decoder/harness.c +119 -0
  213. data/vendor/zstd/doc/educational_decoder/zstd_decompress.c +2323 -0
  214. data/vendor/zstd/doc/educational_decoder/zstd_decompress.h +61 -0
  215. data/vendor/zstd/doc/images/CSpeed2.png +0 -0
  216. data/vendor/zstd/doc/images/DCspeed5.png +0 -0
  217. data/vendor/zstd/doc/images/DSpeed3.png +0 -0
  218. data/vendor/zstd/doc/images/cdict_v136.png +0 -0
  219. data/vendor/zstd/doc/images/dict-cr.png +0 -0
  220. data/vendor/zstd/doc/images/dict-cs.png +0 -0
  221. data/vendor/zstd/doc/images/dict-ds.png +0 -0
  222. data/vendor/zstd/doc/images/zstd_cdict_v1_3_5.png +0 -0
  223. data/vendor/zstd/doc/images/zstd_logo86.png +0 -0
  224. data/vendor/zstd/doc/zstd_compression_format.md +1772 -0
  225. data/vendor/zstd/doc/zstd_manual.html +2244 -0
  226. data/vendor/zstd/examples/Makefile +93 -0
  227. data/vendor/zstd/examples/README.md +46 -0
  228. data/vendor/zstd/examples/common.h +246 -0
  229. data/vendor/zstd/examples/dictionary_compression.c +107 -0
  230. data/vendor/zstd/examples/dictionary_decompression.c +99 -0
  231. data/vendor/zstd/examples/multiple_simple_compression.c +116 -0
  232. data/vendor/zstd/examples/multiple_streaming_compression.c +133 -0
  233. data/vendor/zstd/examples/simple_compression.c +68 -0
  234. data/vendor/zstd/examples/simple_decompression.c +65 -0
  235. data/vendor/zstd/examples/streaming_compression.c +146 -0
  236. data/vendor/zstd/examples/streaming_compression_thread_pool.c +180 -0
  237. data/vendor/zstd/examples/streaming_decompression.c +100 -0
  238. data/vendor/zstd/examples/streaming_memory_usage.c +137 -0
  239. data/vendor/zstd/lib/BUCK +232 -0
  240. data/vendor/zstd/lib/Makefile +389 -0
  241. data/vendor/zstd/lib/README.md +267 -0
  242. data/vendor/zstd/lib/common/allocations.h +55 -0
  243. data/vendor/zstd/lib/common/bits.h +205 -0
  244. data/vendor/zstd/lib/common/bitstream.h +454 -0
  245. data/vendor/zstd/lib/common/compiler.h +482 -0
  246. data/vendor/zstd/lib/common/cpu.h +249 -0
  247. data/vendor/zstd/lib/common/debug.c +30 -0
  248. data/vendor/zstd/lib/common/debug.h +107 -0
  249. data/vendor/zstd/lib/common/debug.o +0 -0
  250. data/vendor/zstd/lib/common/entropy_common.c +340 -0
  251. data/vendor/zstd/lib/common/entropy_common.o +0 -0
  252. data/vendor/zstd/lib/common/error_private.c +64 -0
  253. data/vendor/zstd/lib/common/error_private.h +158 -0
  254. data/vendor/zstd/lib/common/error_private.o +0 -0
  255. data/vendor/zstd/lib/common/fse.h +625 -0
  256. data/vendor/zstd/lib/common/fse_decompress.c +315 -0
  257. data/vendor/zstd/lib/common/fse_decompress.o +0 -0
  258. data/vendor/zstd/lib/common/huf.h +277 -0
  259. data/vendor/zstd/lib/common/mem.h +422 -0
  260. data/vendor/zstd/lib/common/pool.c +371 -0
  261. data/vendor/zstd/lib/common/pool.h +81 -0
  262. data/vendor/zstd/lib/common/pool.o +0 -0
  263. data/vendor/zstd/lib/common/portability_macros.h +190 -0
  264. data/vendor/zstd/lib/common/threading.c +196 -0
  265. data/vendor/zstd/lib/common/threading.h +142 -0
  266. data/vendor/zstd/lib/common/threading.o +0 -0
  267. data/vendor/zstd/lib/common/xxhash.c +18 -0
  268. data/vendor/zstd/lib/common/xxhash.h +7094 -0
  269. data/vendor/zstd/lib/common/xxhash.o +0 -0
  270. data/vendor/zstd/lib/common/zstd_common.c +57 -0
  271. data/vendor/zstd/lib/common/zstd_common.o +0 -0
  272. data/vendor/zstd/lib/common/zstd_deps.h +123 -0
  273. data/vendor/zstd/lib/common/zstd_internal.h +326 -0
  274. data/vendor/zstd/lib/common/zstd_trace.h +156 -0
  275. data/vendor/zstd/lib/compress/clevels.h +134 -0
  276. data/vendor/zstd/lib/compress/fse_compress.c +625 -0
  277. data/vendor/zstd/lib/compress/fse_compress.o +0 -0
  278. data/vendor/zstd/lib/compress/hist.c +446 -0
  279. data/vendor/zstd/lib/compress/hist.h +86 -0
  280. data/vendor/zstd/lib/compress/hist.o +0 -0
  281. data/vendor/zstd/lib/compress/huf_compress.c +1465 -0
  282. data/vendor/zstd/lib/compress/huf_compress.o +0 -0
  283. data/vendor/zstd/lib/compress/zstd_compress.c +8362 -0
  284. data/vendor/zstd/lib/compress/zstd_compress.o +0 -0
  285. data/vendor/zstd/lib/compress/zstd_compress_internal.h +1636 -0
  286. data/vendor/zstd/lib/compress/zstd_compress_literals.c +235 -0
  287. data/vendor/zstd/lib/compress/zstd_compress_literals.h +39 -0
  288. data/vendor/zstd/lib/compress/zstd_compress_literals.o +0 -0
  289. data/vendor/zstd/lib/compress/zstd_compress_sequences.c +442 -0
  290. data/vendor/zstd/lib/compress/zstd_compress_sequences.h +55 -0
  291. data/vendor/zstd/lib/compress/zstd_compress_sequences.o +0 -0
  292. data/vendor/zstd/lib/compress/zstd_compress_superblock.c +688 -0
  293. data/vendor/zstd/lib/compress/zstd_compress_superblock.h +32 -0
  294. data/vendor/zstd/lib/compress/zstd_compress_superblock.o +0 -0
  295. data/vendor/zstd/lib/compress/zstd_cwksp.h +765 -0
  296. data/vendor/zstd/lib/compress/zstd_double_fast.c +778 -0
  297. data/vendor/zstd/lib/compress/zstd_double_fast.h +42 -0
  298. data/vendor/zstd/lib/compress/zstd_double_fast.o +0 -0
  299. data/vendor/zstd/lib/compress/zstd_fast.c +985 -0
  300. data/vendor/zstd/lib/compress/zstd_fast.h +30 -0
  301. data/vendor/zstd/lib/compress/zstd_fast.o +0 -0
  302. data/vendor/zstd/lib/compress/zstd_lazy.c +2243 -0
  303. data/vendor/zstd/lib/compress/zstd_lazy.h +193 -0
  304. data/vendor/zstd/lib/compress/zstd_lazy.o +0 -0
  305. data/vendor/zstd/lib/compress/zstd_ldm.c +745 -0
  306. data/vendor/zstd/lib/compress/zstd_ldm.h +109 -0
  307. data/vendor/zstd/lib/compress/zstd_ldm.o +0 -0
  308. data/vendor/zstd/lib/compress/zstd_ldm_geartab.h +106 -0
  309. data/vendor/zstd/lib/compress/zstd_opt.c +1572 -0
  310. data/vendor/zstd/lib/compress/zstd_opt.h +72 -0
  311. data/vendor/zstd/lib/compress/zstd_opt.o +0 -0
  312. data/vendor/zstd/lib/compress/zstd_preSplit.c +238 -0
  313. data/vendor/zstd/lib/compress/zstd_preSplit.h +33 -0
  314. data/vendor/zstd/lib/compress/zstd_preSplit.o +0 -0
  315. data/vendor/zstd/lib/compress/zstdmt_compress.c +1924 -0
  316. data/vendor/zstd/lib/compress/zstdmt_compress.h +102 -0
  317. data/vendor/zstd/lib/compress/zstdmt_compress.o +0 -0
  318. data/vendor/zstd/lib/decompress/huf_decompress.c +1953 -0
  319. data/vendor/zstd/lib/decompress/huf_decompress.o +0 -0
  320. data/vendor/zstd/lib/decompress/huf_decompress_amd64.S +766 -0
  321. data/vendor/zstd/lib/decompress/zstd_ddict.c +244 -0
  322. data/vendor/zstd/lib/decompress/zstd_ddict.h +44 -0
  323. data/vendor/zstd/lib/decompress/zstd_ddict.o +0 -0
  324. data/vendor/zstd/lib/decompress/zstd_decompress.c +2410 -0
  325. data/vendor/zstd/lib/decompress/zstd_decompress.o +0 -0
  326. data/vendor/zstd/lib/decompress/zstd_decompress_block.c +2311 -0
  327. data/vendor/zstd/lib/decompress/zstd_decompress_block.h +73 -0
  328. data/vendor/zstd/lib/decompress/zstd_decompress_block.o +0 -0
  329. data/vendor/zstd/lib/decompress/zstd_decompress_internal.h +240 -0
  330. data/vendor/zstd/lib/deprecated/zbuff.h +214 -0
  331. data/vendor/zstd/lib/deprecated/zbuff_common.c +26 -0
  332. data/vendor/zstd/lib/deprecated/zbuff_compress.c +167 -0
  333. data/vendor/zstd/lib/deprecated/zbuff_decompress.c +77 -0
  334. data/vendor/zstd/lib/dictBuilder/cover.c +1333 -0
  335. data/vendor/zstd/lib/dictBuilder/cover.h +152 -0
  336. data/vendor/zstd/lib/dictBuilder/divsufsort.c +1913 -0
  337. data/vendor/zstd/lib/dictBuilder/divsufsort.h +57 -0
  338. data/vendor/zstd/lib/dictBuilder/fastcover.c +765 -0
  339. data/vendor/zstd/lib/dictBuilder/zdict.c +1137 -0
  340. data/vendor/zstd/lib/dll/example/Makefile +48 -0
  341. data/vendor/zstd/lib/dll/example/README.md +63 -0
  342. data/vendor/zstd/lib/dll/example/build_package.bat +55 -0
  343. data/vendor/zstd/lib/dll/example/fullbench-dll.sln +25 -0
  344. data/vendor/zstd/lib/dll/example/fullbench-dll.vcxproj +181 -0
  345. data/vendor/zstd/lib/install_oses.mk +17 -0
  346. data/vendor/zstd/lib/legacy/zstd_legacy.h +452 -0
  347. data/vendor/zstd/lib/legacy/zstd_v01.c +2128 -0
  348. data/vendor/zstd/lib/legacy/zstd_v01.h +94 -0
  349. data/vendor/zstd/lib/legacy/zstd_v02.c +3465 -0
  350. data/vendor/zstd/lib/legacy/zstd_v02.h +93 -0
  351. data/vendor/zstd/lib/legacy/zstd_v03.c +3105 -0
  352. data/vendor/zstd/lib/legacy/zstd_v03.h +93 -0
  353. data/vendor/zstd/lib/legacy/zstd_v04.c +3598 -0
  354. data/vendor/zstd/lib/legacy/zstd_v04.h +142 -0
  355. data/vendor/zstd/lib/legacy/zstd_v05.c +4005 -0
  356. data/vendor/zstd/lib/legacy/zstd_v05.h +162 -0
  357. data/vendor/zstd/lib/legacy/zstd_v06.c +4110 -0
  358. data/vendor/zstd/lib/legacy/zstd_v06.h +172 -0
  359. data/vendor/zstd/lib/legacy/zstd_v07.c +4490 -0
  360. data/vendor/zstd/lib/legacy/zstd_v07.h +187 -0
  361. data/vendor/zstd/lib/libzstd.mk +238 -0
  362. data/vendor/zstd/lib/libzstd.pc.in +17 -0
  363. data/vendor/zstd/lib/module.modulemap +35 -0
  364. data/vendor/zstd/lib/zdict.h +481 -0
  365. data/vendor/zstd/lib/zstd.h +3209 -0
  366. data/vendor/zstd/lib/zstd_errors.h +107 -0
  367. data/vendor/zstd/programs/BUCK +44 -0
  368. data/vendor/zstd/programs/Makefile +445 -0
  369. data/vendor/zstd/programs/README.md +344 -0
  370. data/vendor/zstd/programs/benchfn.c +256 -0
  371. data/vendor/zstd/programs/benchfn.h +173 -0
  372. data/vendor/zstd/programs/benchzstd.c +1270 -0
  373. data/vendor/zstd/programs/benchzstd.h +191 -0
  374. data/vendor/zstd/programs/datagen.c +186 -0
  375. data/vendor/zstd/programs/datagen.h +38 -0
  376. data/vendor/zstd/programs/dibio.c +447 -0
  377. data/vendor/zstd/programs/dibio.h +39 -0
  378. data/vendor/zstd/programs/fileio.c +3717 -0
  379. data/vendor/zstd/programs/fileio.h +171 -0
  380. data/vendor/zstd/programs/fileio_asyncio.c +663 -0
  381. data/vendor/zstd/programs/fileio_asyncio.h +195 -0
  382. data/vendor/zstd/programs/fileio_common.h +121 -0
  383. data/vendor/zstd/programs/fileio_types.h +86 -0
  384. data/vendor/zstd/programs/lorem.c +285 -0
  385. data/vendor/zstd/programs/lorem.h +32 -0
  386. data/vendor/zstd/programs/platform.h +217 -0
  387. data/vendor/zstd/programs/timefn.c +170 -0
  388. data/vendor/zstd/programs/timefn.h +59 -0
  389. data/vendor/zstd/programs/util.c +1731 -0
  390. data/vendor/zstd/programs/util.h +364 -0
  391. data/vendor/zstd/programs/windres/verrsrc.h +17 -0
  392. data/vendor/zstd/programs/windres/zstd.rc +51 -0
  393. data/vendor/zstd/programs/windres/zstd32.res +0 -0
  394. data/vendor/zstd/programs/windres/zstd64.res +0 -0
  395. data/vendor/zstd/programs/zstd.1 +580 -0
  396. data/vendor/zstd/programs/zstd.1.md +714 -0
  397. data/vendor/zstd/programs/zstdcli.c +1675 -0
  398. data/vendor/zstd/programs/zstdcli_trace.c +173 -0
  399. data/vendor/zstd/programs/zstdcli_trace.h +24 -0
  400. data/vendor/zstd/programs/zstdgrep +134 -0
  401. data/vendor/zstd/programs/zstdgrep.1 +26 -0
  402. data/vendor/zstd/programs/zstdgrep.1.md +30 -0
  403. data/vendor/zstd/programs/zstdless +8 -0
  404. data/vendor/zstd/programs/zstdless.1 +14 -0
  405. data/vendor/zstd/programs/zstdless.1.md +16 -0
  406. data/vendor/zstd/tests/DEPRECATED-test-zstd-speed.py +378 -0
  407. data/vendor/zstd/tests/Makefile +485 -0
  408. data/vendor/zstd/tests/README.md +184 -0
  409. data/vendor/zstd/tests/automated_benchmarking.py +326 -0
  410. data/vendor/zstd/tests/checkTag.c +65 -0
  411. data/vendor/zstd/tests/check_size.py +31 -0
  412. data/vendor/zstd/tests/cli-tests/README.md +258 -0
  413. data/vendor/zstd/tests/cli-tests/basic/args.sh +10 -0
  414. data/vendor/zstd/tests/cli-tests/basic/args.sh.exit +1 -0
  415. data/vendor/zstd/tests/cli-tests/basic/args.sh.stderr.glob +28 -0
  416. data/vendor/zstd/tests/cli-tests/basic/help.sh +10 -0
  417. data/vendor/zstd/tests/cli-tests/basic/help.sh.stdout.glob +34 -0
  418. data/vendor/zstd/tests/cli-tests/basic/memlimit.sh +40 -0
  419. data/vendor/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact +13 -0
  420. data/vendor/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact +13 -0
  421. data/vendor/zstd/tests/cli-tests/basic/output_dir.sh +7 -0
  422. data/vendor/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact +2 -0
  423. data/vendor/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact +2 -0
  424. data/vendor/zstd/tests/cli-tests/basic/version.sh +6 -0
  425. data/vendor/zstd/tests/cli-tests/basic/version.sh.stdout.glob +2 -0
  426. data/vendor/zstd/tests/cli-tests/bin/cmp_size +44 -0
  427. data/vendor/zstd/tests/cli-tests/bin/datagen +3 -0
  428. data/vendor/zstd/tests/cli-tests/bin/die +4 -0
  429. data/vendor/zstd/tests/cli-tests/bin/println +2 -0
  430. data/vendor/zstd/tests/cli-tests/bin/unzstd +1 -0
  431. data/vendor/zstd/tests/cli-tests/bin/zstd +9 -0
  432. data/vendor/zstd/tests/cli-tests/bin/zstdcat +1 -0
  433. data/vendor/zstd/tests/cli-tests/bin/zstdgrep +2 -0
  434. data/vendor/zstd/tests/cli-tests/bin/zstdless +2 -0
  435. data/vendor/zstd/tests/cli-tests/cltools/setup +6 -0
  436. data/vendor/zstd/tests/cli-tests/cltools/zstdgrep.sh +8 -0
  437. data/vendor/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit +1 -0
  438. data/vendor/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact +1 -0
  439. data/vendor/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob +4 -0
  440. data/vendor/zstd/tests/cli-tests/cltools/zstdless.sh +10 -0
  441. data/vendor/zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact +2 -0
  442. data/vendor/zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob +5 -0
  443. data/vendor/zstd/tests/cli-tests/common/format.sh +19 -0
  444. data/vendor/zstd/tests/cli-tests/common/mtime.sh +13 -0
  445. data/vendor/zstd/tests/cli-tests/common/permissions.sh +18 -0
  446. data/vendor/zstd/tests/cli-tests/common/platform.sh +47 -0
  447. data/vendor/zstd/tests/cli-tests/compression/adapt.sh +14 -0
  448. data/vendor/zstd/tests/cli-tests/compression/basic.sh +36 -0
  449. data/vendor/zstd/tests/cli-tests/compression/compress-literals.sh +10 -0
  450. data/vendor/zstd/tests/cli-tests/compression/format.sh +16 -0
  451. data/vendor/zstd/tests/cli-tests/compression/golden.sh +16 -0
  452. data/vendor/zstd/tests/cli-tests/compression/gzip-compat.sh +17 -0
  453. data/vendor/zstd/tests/cli-tests/compression/levels.sh +75 -0
  454. data/vendor/zstd/tests/cli-tests/compression/levels.sh.stderr.exact +80 -0
  455. data/vendor/zstd/tests/cli-tests/compression/long-distance-matcher.sh +7 -0
  456. data/vendor/zstd/tests/cli-tests/compression/multi-threaded.sh +22 -0
  457. data/vendor/zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact +21 -0
  458. data/vendor/zstd/tests/cli-tests/compression/multiple-files.sh +21 -0
  459. data/vendor/zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact +12 -0
  460. data/vendor/zstd/tests/cli-tests/compression/row-match-finder.sh +7 -0
  461. data/vendor/zstd/tests/cli-tests/compression/setup +7 -0
  462. data/vendor/zstd/tests/cli-tests/compression/stream-size.sh +7 -0
  463. data/vendor/zstd/tests/cli-tests/compression/verbose-wlog.sh +11 -0
  464. data/vendor/zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob +5 -0
  465. data/vendor/zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob +5 -0
  466. data/vendor/zstd/tests/cli-tests/compression/window-resize.sh +9 -0
  467. data/vendor/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore +0 -0
  468. data/vendor/zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob +3 -0
  469. data/vendor/zstd/tests/cli-tests/decompression/detectErrors.sh +11 -0
  470. data/vendor/zstd/tests/cli-tests/decompression/golden.sh +7 -0
  471. data/vendor/zstd/tests/cli-tests/decompression/pass-through.sh +57 -0
  472. data/vendor/zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact +11 -0
  473. data/vendor/zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact +25 -0
  474. data/vendor/zstd/tests/cli-tests/determinism/basic.sh +36 -0
  475. data/vendor/zstd/tests/cli-tests/determinism/basic.sh.stderr.exact +0 -0
  476. data/vendor/zstd/tests/cli-tests/determinism/basic.sh.stdout.exact +880 -0
  477. data/vendor/zstd/tests/cli-tests/determinism/multithread.sh +45 -0
  478. data/vendor/zstd/tests/cli-tests/determinism/multithread.sh.stderr.exact +0 -0
  479. data/vendor/zstd/tests/cli-tests/determinism/multithread.sh.stdout.exact +260 -0
  480. data/vendor/zstd/tests/cli-tests/determinism/reuse.sh +44 -0
  481. data/vendor/zstd/tests/cli-tests/determinism/reuse.sh.stderr.exact +0 -0
  482. data/vendor/zstd/tests/cli-tests/determinism/reuse.sh.stdout.exact +19 -0
  483. data/vendor/zstd/tests/cli-tests/determinism/setup +5 -0
  484. data/vendor/zstd/tests/cli-tests/determinism/setup_once +30 -0
  485. data/vendor/zstd/tests/cli-tests/dict-builder/empty-input.sh +9 -0
  486. data/vendor/zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact +1 -0
  487. data/vendor/zstd/tests/cli-tests/dict-builder/no-inputs.sh +3 -0
  488. data/vendor/zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit +1 -0
  489. data/vendor/zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact +5 -0
  490. data/vendor/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh +29 -0
  491. data/vendor/zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact +7 -0
  492. data/vendor/zstd/tests/cli-tests/dictionaries/golden.sh +9 -0
  493. data/vendor/zstd/tests/cli-tests/dictionaries/setup +6 -0
  494. data/vendor/zstd/tests/cli-tests/dictionaries/setup_once +24 -0
  495. data/vendor/zstd/tests/cli-tests/file-handling/directory-mirror.sh +49 -0
  496. data/vendor/zstd/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact +0 -0
  497. data/vendor/zstd/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact +0 -0
  498. data/vendor/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh +12 -0
  499. data/vendor/zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact +30 -0
  500. data/vendor/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh +9 -0
  501. data/vendor/zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.glob +46 -0
  502. data/vendor/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh +8 -0
  503. data/vendor/zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact +24 -0
  504. data/vendor/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh +8 -0
  505. data/vendor/zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.glob +28 -0
  506. data/vendor/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh +8 -0
  507. data/vendor/zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact +18 -0
  508. data/vendor/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh +8 -0
  509. data/vendor/zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.glob +42 -0
  510. data/vendor/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh +7 -0
  511. data/vendor/zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact +18 -0
  512. data/vendor/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh +7 -0
  513. data/vendor/zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.glob +24 -0
  514. data/vendor/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh +7 -0
  515. data/vendor/zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact +14 -0
  516. data/vendor/zstd/tests/cli-tests/progress/no-progress.sh +46 -0
  517. data/vendor/zstd/tests/cli-tests/progress/no-progress.sh.stderr.glob +96 -0
  518. data/vendor/zstd/tests/cli-tests/progress/progress.sh +41 -0
  519. data/vendor/zstd/tests/cli-tests/progress/progress.sh.stderr.glob +62 -0
  520. data/vendor/zstd/tests/cli-tests/run.py +732 -0
  521. data/vendor/zstd/tests/cli-tests/zstd-symlinks/setup +6 -0
  522. data/vendor/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh +12 -0
  523. data/vendor/zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact +8 -0
  524. data/vendor/zstd/tests/datagencli.c +149 -0
  525. data/vendor/zstd/tests/decodecorpus.c +1998 -0
  526. data/vendor/zstd/tests/dict-files/zero-weight-dict +0 -0
  527. data/vendor/zstd/tests/external_matchfinder.c +140 -0
  528. data/vendor/zstd/tests/external_matchfinder.h +39 -0
  529. data/vendor/zstd/tests/fullbench.c +1210 -0
  530. data/vendor/zstd/tests/fuzz/Makefile +278 -0
  531. data/vendor/zstd/tests/fuzz/README.md +161 -0
  532. data/vendor/zstd/tests/fuzz/block_decompress.c +53 -0
  533. data/vendor/zstd/tests/fuzz/block_round_trip.c +103 -0
  534. data/vendor/zstd/tests/fuzz/decompress_cross_format.c +130 -0
  535. data/vendor/zstd/tests/fuzz/decompress_dstSize_tooSmall.c +74 -0
  536. data/vendor/zstd/tests/fuzz/dictionary_decompress.c +77 -0
  537. data/vendor/zstd/tests/fuzz/dictionary_loader.c +106 -0
  538. data/vendor/zstd/tests/fuzz/dictionary_round_trip.c +155 -0
  539. data/vendor/zstd/tests/fuzz/dictionary_stream_round_trip.c +209 -0
  540. data/vendor/zstd/tests/fuzz/fse_read_ncount.c +100 -0
  541. data/vendor/zstd/tests/fuzz/fuzz.h +57 -0
  542. data/vendor/zstd/tests/fuzz/fuzz.py +910 -0
  543. data/vendor/zstd/tests/fuzz/fuzz_data_producer.c +95 -0
  544. data/vendor/zstd/tests/fuzz/fuzz_data_producer.h +66 -0
  545. data/vendor/zstd/tests/fuzz/fuzz_helpers.c +47 -0
  546. data/vendor/zstd/tests/fuzz/fuzz_helpers.h +82 -0
  547. data/vendor/zstd/tests/fuzz/fuzz_third_party_seq_prod.h +116 -0
  548. data/vendor/zstd/tests/fuzz/generate_sequences.c +88 -0
  549. data/vendor/zstd/tests/fuzz/huf_decompress.c +68 -0
  550. data/vendor/zstd/tests/fuzz/huf_round_trip.c +137 -0
  551. data/vendor/zstd/tests/fuzz/raw_dictionary_round_trip.c +119 -0
  552. data/vendor/zstd/tests/fuzz/regression_driver.c +90 -0
  553. data/vendor/zstd/tests/fuzz/seekable_roundtrip.c +88 -0
  554. data/vendor/zstd/tests/fuzz/seq_prod_fuzz_example/Makefile +16 -0
  555. data/vendor/zstd/tests/fuzz/seq_prod_fuzz_example/README.md +12 -0
  556. data/vendor/zstd/tests/fuzz/seq_prod_fuzz_example/example_seq_prod.c +52 -0
  557. data/vendor/zstd/tests/fuzz/sequence_compression_api.c +452 -0
  558. data/vendor/zstd/tests/fuzz/simple_compress.c +60 -0
  559. data/vendor/zstd/tests/fuzz/simple_decompress.c +59 -0
  560. data/vendor/zstd/tests/fuzz/simple_round_trip.c +182 -0
  561. data/vendor/zstd/tests/fuzz/stream_decompress.c +119 -0
  562. data/vendor/zstd/tests/fuzz/stream_round_trip.c +218 -0
  563. data/vendor/zstd/tests/fuzz/zstd_frame_info.c +43 -0
  564. data/vendor/zstd/tests/fuzz/zstd_helpers.c +208 -0
  565. data/vendor/zstd/tests/fuzz/zstd_helpers.h +56 -0
  566. data/vendor/zstd/tests/fuzzer.c +5482 -0
  567. data/vendor/zstd/tests/golden-compression/PR-3517-block-splitter-corruption-test +1 -0
  568. data/vendor/zstd/tests/golden-compression/http +1 -0
  569. data/vendor/zstd/tests/golden-compression/huffman-compressed-larger +0 -0
  570. data/vendor/zstd/tests/golden-compression/large-literal-and-match-lengths +0 -0
  571. data/vendor/zstd/tests/golden-decompression/block-128k.zst +0 -0
  572. data/vendor/zstd/tests/golden-decompression/empty-block.zst +0 -0
  573. data/vendor/zstd/tests/golden-decompression/rle-first-block.zst +0 -0
  574. data/vendor/zstd/tests/golden-decompression/zeroSeq_2B.zst +0 -0
  575. data/vendor/zstd/tests/golden-decompression-errors/off0.bin.zst +0 -0
  576. data/vendor/zstd/tests/golden-decompression-errors/truncated_huff_state.zst +0 -0
  577. data/vendor/zstd/tests/golden-decompression-errors/zeroSeq_extraneous.zst +0 -0
  578. data/vendor/zstd/tests/golden-dictionaries/http-dict-missing-symbols +0 -0
  579. data/vendor/zstd/tests/gzip/Makefile +45 -0
  580. data/vendor/zstd/tests/gzip/gzip-env.sh +46 -0
  581. data/vendor/zstd/tests/gzip/helin-segv.sh +31 -0
  582. data/vendor/zstd/tests/gzip/help-version.sh +270 -0
  583. data/vendor/zstd/tests/gzip/hufts-segv.gz +0 -0
  584. data/vendor/zstd/tests/gzip/hufts.sh +34 -0
  585. data/vendor/zstd/tests/gzip/init.cfg +5 -0
  586. data/vendor/zstd/tests/gzip/init.sh +616 -0
  587. data/vendor/zstd/tests/gzip/keep.sh +51 -0
  588. data/vendor/zstd/tests/gzip/list.sh +31 -0
  589. data/vendor/zstd/tests/gzip/memcpy-abuse.sh +34 -0
  590. data/vendor/zstd/tests/gzip/mixed.sh +68 -0
  591. data/vendor/zstd/tests/gzip/null-suffix-clobber.sh +35 -0
  592. data/vendor/zstd/tests/gzip/stdin.sh +31 -0
  593. data/vendor/zstd/tests/gzip/test-driver.sh +150 -0
  594. data/vendor/zstd/tests/gzip/trailing-nul.sh +37 -0
  595. data/vendor/zstd/tests/gzip/unpack-invalid.sh +36 -0
  596. data/vendor/zstd/tests/gzip/z-suffix.sh +30 -0
  597. data/vendor/zstd/tests/gzip/zdiff.sh +48 -0
  598. data/vendor/zstd/tests/gzip/zgrep-context.sh +47 -0
  599. data/vendor/zstd/tests/gzip/zgrep-f.sh +43 -0
  600. data/vendor/zstd/tests/gzip/zgrep-signal.sh +64 -0
  601. data/vendor/zstd/tests/gzip/znew-k.sh +40 -0
  602. data/vendor/zstd/tests/invalidDictionaries.c +61 -0
  603. data/vendor/zstd/tests/largeDictionary.c +131 -0
  604. data/vendor/zstd/tests/legacy.c +262 -0
  605. data/vendor/zstd/tests/libzstd_builds.sh +104 -0
  606. data/vendor/zstd/tests/longmatch.c +123 -0
  607. data/vendor/zstd/tests/loremOut.c +50 -0
  608. data/vendor/zstd/tests/loremOut.h +15 -0
  609. data/vendor/zstd/tests/paramgrill.c +2965 -0
  610. data/vendor/zstd/tests/playTests.sh +1926 -0
  611. data/vendor/zstd/tests/poolTests.c +271 -0
  612. data/vendor/zstd/tests/rateLimiter.py +41 -0
  613. data/vendor/zstd/tests/regression/Makefile +60 -0
  614. data/vendor/zstd/tests/regression/README.md +28 -0
  615. data/vendor/zstd/tests/regression/config.c +404 -0
  616. data/vendor/zstd/tests/regression/config.h +91 -0
  617. data/vendor/zstd/tests/regression/data.c +631 -0
  618. data/vendor/zstd/tests/regression/data.h +121 -0
  619. data/vendor/zstd/tests/regression/levels.h +59 -0
  620. data/vendor/zstd/tests/regression/method.c +701 -0
  621. data/vendor/zstd/tests/regression/method.h +65 -0
  622. data/vendor/zstd/tests/regression/result.c +30 -0
  623. data/vendor/zstd/tests/regression/result.h +103 -0
  624. data/vendor/zstd/tests/regression/results.csv +1480 -0
  625. data/vendor/zstd/tests/regression/test.c +362 -0
  626. data/vendor/zstd/tests/roundTripCrash.c +241 -0
  627. data/vendor/zstd/tests/seqgen.c +260 -0
  628. data/vendor/zstd/tests/seqgen.h +58 -0
  629. data/vendor/zstd/tests/test-license.py +156 -0
  630. data/vendor/zstd/tests/test-variants.sh +115 -0
  631. data/vendor/zstd/tests/test-zstd-versions.py +308 -0
  632. data/vendor/zstd/tests/test_process_substitution.bash +92 -0
  633. data/vendor/zstd/tests/zstreamtest.c +3467 -0
  634. data/vendor/zstd/zlibWrapper/BUCK +22 -0
  635. data/vendor/zstd/zlibWrapper/Makefile +120 -0
  636. data/vendor/zstd/zlibWrapper/README.md +163 -0
  637. data/vendor/zstd/zlibWrapper/examples/example.c +598 -0
  638. data/vendor/zstd/zlibWrapper/examples/example_original.c +599 -0
  639. data/vendor/zstd/zlibWrapper/examples/fitblk.c +254 -0
  640. data/vendor/zstd/zlibWrapper/examples/fitblk_original.c +233 -0
  641. data/vendor/zstd/zlibWrapper/examples/minigzip.c +605 -0
  642. data/vendor/zstd/zlibWrapper/examples/zwrapbench.c +1018 -0
  643. data/vendor/zstd/zlibWrapper/gzclose.c +26 -0
  644. data/vendor/zstd/zlibWrapper/gzcompatibility.h +68 -0
  645. data/vendor/zstd/zlibWrapper/gzguts.h +229 -0
  646. data/vendor/zstd/zlibWrapper/gzlib.c +587 -0
  647. data/vendor/zstd/zlibWrapper/gzread.c +637 -0
  648. data/vendor/zstd/zlibWrapper/gzwrite.c +632 -0
  649. data/vendor/zstd/zlibWrapper/zstd_zlibwrapper.c +1200 -0
  650. data/vendor/zstd/zlibWrapper/zstd_zlibwrapper.h +91 -0
  651. metadata +738 -0
@@ -0,0 +1,1018 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ */
9
+
10
+
11
+ /* *************************************
12
+ * Includes
13
+ ***************************************/
14
+ #include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
15
+ #include <stdlib.h> /* malloc, free */
16
+ #include <string.h> /* memset */
17
+ #include <stdio.h> /* fprintf, fopen, ftello64 */
18
+ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
19
+ #include <ctype.h> /* toupper */
20
+ #include <errno.h> /* errno */
21
+
22
+ #include "timefn.h" /* UTIL_time_t, UTIL_getTime, UTIL_clockSpanMicro, UTIL_waitForNextTick */
23
+ #include "mem.h"
24
+ #define ZSTD_STATIC_LINKING_ONLY
25
+ #include "zstd.h"
26
+ #include "datagen.h" /* RDG_genBuffer */
27
+ #include "xxhash.h"
28
+
29
+ #include "../zstd_zlibwrapper.h"
30
+
31
+
32
+
33
+ /*-************************************
34
+ * Tuning parameters
35
+ **************************************/
36
+ #ifndef ZSTDCLI_CLEVEL_DEFAULT
37
+ # define ZSTDCLI_CLEVEL_DEFAULT 3
38
+ #endif
39
+
40
+
41
+ /*-************************************
42
+ * Constants
43
+ **************************************/
44
+ #define COMPRESSOR_NAME "Zstandard wrapper for zlib command line interface"
45
+ #ifndef ZSTD_VERSION
46
+ # define ZSTD_VERSION "v" ZSTD_VERSION_STRING
47
+ #endif
48
+ #define AUTHOR "Yann Collet"
49
+ #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
50
+
51
+ #ifndef ZSTD_GIT_COMMIT
52
+ # define ZSTD_GIT_COMMIT_STRING ""
53
+ #else
54
+ # define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
55
+ #endif
56
+
57
+ #define NBLOOPS 3
58
+ #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
59
+ #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
60
+ #define COOLPERIOD_SEC 10
61
+
62
+ #define KB *(1 <<10)
63
+ #define MB *(1 <<20)
64
+ #define GB *(1U<<30)
65
+
66
+ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
67
+
68
+ static U32 g_compressibilityDefault = 50;
69
+
70
+
71
+ /* *************************************
72
+ * console display
73
+ ***************************************/
74
+ #define DEFAULT_DISPLAY_LEVEL 2
75
+ #define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
76
+ #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
77
+ static unsigned g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
78
+ static FILE* displayOut;
79
+
80
+ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
81
+ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
82
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
83
+ if (g_displayLevel>=4) fflush(displayOut); } }
84
+ static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
85
+ static clock_t g_time = 0;
86
+
87
+
88
+ /* *************************************
89
+ * Exceptions
90
+ ***************************************/
91
+ #ifndef DEBUG
92
+ # define DEBUG 0
93
+ #endif
94
+ #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
95
+ #define EXM_THROW(error, ...) \
96
+ { \
97
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
98
+ DISPLAYLEVEL(1, "Error %i : ", error); \
99
+ DISPLAYLEVEL(1, __VA_ARGS__); \
100
+ DISPLAYLEVEL(1, "\n"); \
101
+ exit(error); \
102
+ }
103
+
104
+
105
+ /* *************************************
106
+ * Benchmark Parameters
107
+ ***************************************/
108
+ static unsigned g_nbIterations = NBLOOPS;
109
+ static size_t g_blockSize = 0;
110
+ int g_additionalParam = 0;
111
+
112
+ static void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
113
+
114
+ static void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
115
+
116
+ static void BMK_SetNbIterations(unsigned nbLoops)
117
+ {
118
+ g_nbIterations = nbLoops;
119
+ DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
120
+ }
121
+
122
+ static void BMK_SetBlockSize(size_t blockSize)
123
+ {
124
+ g_blockSize = blockSize;
125
+ DISPLAYLEVEL(2, "using blocks of size %u KB \n", (unsigned)(blockSize>>10));
126
+ }
127
+
128
+
129
+ /* ********************************************************
130
+ * Bench functions
131
+ **********************************************************/
132
+ #undef MIN
133
+ #undef MAX
134
+ #define MIN(a,b) ((a)<(b) ? (a) : (b))
135
+ #define MAX(a,b) ((a)>(b) ? (a) : (b))
136
+
137
+ typedef struct
138
+ {
139
+ z_const char* srcPtr;
140
+ size_t srcSize;
141
+ char* cPtr;
142
+ size_t cRoom;
143
+ size_t cSize;
144
+ char* resPtr;
145
+ size_t resSize;
146
+ } blockParam_t;
147
+
148
+ typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor;
149
+
150
+
151
+ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
152
+ const char* displayName, int cLevel,
153
+ const size_t* fileSizes, U32 nbFiles,
154
+ const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor)
155
+ {
156
+ size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
157
+ size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
158
+ U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
159
+ blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
160
+ size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
161
+ void* const compressedBuffer = malloc(maxCompressedSize);
162
+ void* const resultBuffer = malloc(srcSize);
163
+ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
164
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
165
+ U32 nbBlocks;
166
+
167
+ /* checks */
168
+ if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
169
+ EXM_THROW(31, "allocation error : not enough memory");
170
+
171
+ /* init */
172
+ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
173
+
174
+ /* Init blockTable data */
175
+ { z_const char* srcPtr = (z_const char*)srcBuffer;
176
+ char* cPtr = (char*)compressedBuffer;
177
+ char* resPtr = (char*)resultBuffer;
178
+ U32 fileNb;
179
+ for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
180
+ size_t remaining = fileSizes[fileNb];
181
+ U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
182
+ U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
183
+ for ( ; nbBlocks<blockEnd; nbBlocks++) {
184
+ size_t const thisBlockSize = MIN(remaining, blockSize);
185
+ blockTable[nbBlocks].srcPtr = srcPtr;
186
+ blockTable[nbBlocks].cPtr = cPtr;
187
+ blockTable[nbBlocks].resPtr = resPtr;
188
+ blockTable[nbBlocks].srcSize = thisBlockSize;
189
+ blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
190
+ srcPtr += thisBlockSize;
191
+ cPtr += blockTable[nbBlocks].cRoom;
192
+ resPtr += thisBlockSize;
193
+ remaining -= thisBlockSize;
194
+ } } }
195
+
196
+ /* warming up memory */
197
+ RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
198
+
199
+ /* Bench */
200
+ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
201
+ U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
202
+ UTIL_time_t coolTime;
203
+ U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
204
+ U64 totalCTime=0, totalDTime=0;
205
+ U32 cCompleted=0, dCompleted=0;
206
+ # define NB_MARKS 4
207
+ const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
208
+ U32 markNb = 0;
209
+ size_t cSize = 0;
210
+ double ratio = 0.;
211
+
212
+ coolTime = UTIL_getTime();
213
+ DISPLAYLEVEL(2, "\r%79s\r", "");
214
+ while (!cCompleted | !dCompleted) {
215
+ UTIL_time_t clockStart;
216
+ U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
217
+
218
+ /* overheat protection */
219
+ if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
220
+ DISPLAYLEVEL(2, "\rcooling down ... \r");
221
+ UTIL_sleep(COOLPERIOD_SEC);
222
+ coolTime = UTIL_getTime();
223
+ }
224
+
225
+ /* Compression */
226
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize);
227
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
228
+
229
+ UTIL_sleepMilli(1); /* give processor time to other processes */
230
+ UTIL_waitForNextTick();
231
+ clockStart = UTIL_getTime();
232
+
233
+ if (!cCompleted) { /* still some time to do compression tests */
234
+ U32 nbLoops = 0;
235
+ if (compressor == BMK_ZSTD) {
236
+ ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
237
+ ZSTD_customMem const cmem = { NULL, NULL, NULL };
238
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_auto, zparams.cParams, cmem);
239
+ if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
240
+
241
+ do {
242
+ U32 blockNb;
243
+ size_t rSize;
244
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
245
+ if (dictBufferSize) {
246
+ rSize = ZSTD_compress_usingCDict(ctx,
247
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
248
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
249
+ cdict);
250
+ } else {
251
+ rSize = ZSTD_compressCCtx (ctx,
252
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
253
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
254
+ }
255
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
256
+ blockTable[blockNb].cSize = rSize;
257
+ }
258
+ nbLoops++;
259
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
260
+ ZSTD_freeCDict(cdict);
261
+ } else if (compressor == BMK_ZSTD_STREAM) {
262
+ ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
263
+ ZSTD_inBuffer inBuffer;
264
+ ZSTD_outBuffer outBuffer;
265
+ ZSTD_CStream* zbc = ZSTD_createCStream();
266
+ size_t rSize;
267
+ ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
268
+
269
+ if (!cctxParams) EXM_THROW(1, "ZSTD_createCCtxParams() allocation failure");
270
+ if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure");
271
+
272
+ { int initErr = 0;
273
+ initErr |= ZSTD_isError(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only));
274
+ initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, zparams));
275
+ initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zbc, cctxParams));
276
+ initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zbc, avgSize));
277
+ initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zbc, dictBuffer, dictBufferSize));
278
+
279
+ ZSTD_freeCCtxParams(cctxParams);
280
+ if (initErr) EXM_THROW(1, "CCtx init failed!");
281
+ }
282
+
283
+ do {
284
+ U32 blockNb;
285
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
286
+ rSize = ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only);
287
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_CCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
288
+ rSize = ZSTD_CCtx_setPledgedSrcSize(zbc, blockTable[blockNb].srcSize);
289
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_CCtx_setPledgedSrcSize() failed : %s", ZSTD_getErrorName(rSize));
290
+ inBuffer.src = blockTable[blockNb].srcPtr;
291
+ inBuffer.size = blockTable[blockNb].srcSize;
292
+ inBuffer.pos = 0;
293
+ outBuffer.dst = blockTable[blockNb].cPtr;
294
+ outBuffer.size = blockTable[blockNb].cRoom;
295
+ outBuffer.pos = 0;
296
+ rSize = ZSTD_compressStream(zbc, &outBuffer, &inBuffer);
297
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compressStream() failed : %s", ZSTD_getErrorName(rSize));
298
+ rSize = ZSTD_endStream(zbc, &outBuffer);
299
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_endStream() failed : %s", ZSTD_getErrorName(rSize));
300
+ blockTable[blockNb].cSize = outBuffer.pos;
301
+ }
302
+ nbLoops++;
303
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
304
+ ZSTD_freeCStream(zbc);
305
+ } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
306
+ z_stream def;
307
+ int ret;
308
+ int useSetDict = (dictBuffer != NULL);
309
+ if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) ZWRAP_useZSTDcompression(0);
310
+ else ZWRAP_useZSTDcompression(1);
311
+ def.zalloc = Z_NULL;
312
+ def.zfree = Z_NULL;
313
+ def.opaque = Z_NULL;
314
+ ret = deflateInit(&def, cLevel);
315
+ if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
316
+ /* if (ZWRAP_isUsingZSTDcompression()) {
317
+ ret = ZWRAP_setPledgedSrcSize(&def, avgSize);
318
+ if (ret != Z_OK) EXM_THROW(1, "ZWRAP_setPledgedSrcSize failure");
319
+ } */
320
+ do {
321
+ U32 blockNb;
322
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
323
+ if (ZWRAP_isUsingZSTDcompression())
324
+ ret = ZWRAP_deflateReset_keepDict(&def); /* reuse dictionary to make compression faster */
325
+ else
326
+ ret = deflateReset(&def);
327
+ if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
328
+ if (useSetDict) {
329
+ ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
330
+ if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
331
+ if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
332
+ }
333
+ def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
334
+ def.avail_in = (uInt)blockTable[blockNb].srcSize;
335
+ def.total_in = 0;
336
+ def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
337
+ def.avail_out = (uInt)blockTable[blockNb].cRoom;
338
+ def.total_out = 0;
339
+ ret = deflate(&def, Z_FINISH);
340
+ if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize);
341
+ blockTable[blockNb].cSize = def.total_out;
342
+ }
343
+ nbLoops++;
344
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
345
+ ret = deflateEnd(&def);
346
+ if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
347
+ } else {
348
+ z_stream def;
349
+ if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) ZWRAP_useZSTDcompression(0);
350
+ else ZWRAP_useZSTDcompression(1);
351
+ do {
352
+ U32 blockNb;
353
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
354
+ int ret;
355
+ def.zalloc = Z_NULL;
356
+ def.zfree = Z_NULL;
357
+ def.opaque = Z_NULL;
358
+ ret = deflateInit(&def, cLevel);
359
+ if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
360
+ if (dictBuffer) {
361
+ ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize);
362
+ if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
363
+ }
364
+ def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr;
365
+ def.avail_in = (uInt)blockTable[blockNb].srcSize;
366
+ def.total_in = 0;
367
+ def.next_out = (z_Bytef*) blockTable[blockNb].cPtr;
368
+ def.avail_out = (uInt)blockTable[blockNb].cRoom;
369
+ def.total_out = 0;
370
+ ret = deflate(&def, Z_FINISH);
371
+ if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure");
372
+ ret = deflateEnd(&def);
373
+ if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
374
+ blockTable[blockNb].cSize = def.total_out;
375
+ }
376
+ nbLoops++;
377
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
378
+ }
379
+ { U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
380
+ if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
381
+ totalCTime += clockSpan;
382
+ cCompleted = totalCTime>maxTime;
383
+ } }
384
+
385
+ cSize = 0;
386
+ { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
387
+ ratio = (double)srcSize / (double)cSize;
388
+ markNb = (markNb+1) % NB_MARKS;
389
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
390
+ marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
391
+ (double)srcSize / (double)fastestC );
392
+
393
+ (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
394
+ #if 1
395
+ /* Decompression */
396
+ if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
397
+
398
+ UTIL_sleepMilli(1); /* give processor time to other processes */
399
+ UTIL_waitForNextTick();
400
+ clockStart = UTIL_getTime();
401
+
402
+ if (!dCompleted) {
403
+ U32 nbLoops = 0;
404
+ if (compressor == BMK_ZSTD) {
405
+ ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
406
+ if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
407
+ do {
408
+ unsigned blockNb;
409
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
410
+ size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
411
+ blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
412
+ blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
413
+ ddict);
414
+ if (ZSTD_isError(regenSize)) {
415
+ DISPLAY("ZSTD_decompress_usingDDict() failed on block %u : %s \n",
416
+ blockNb, ZSTD_getErrorName(regenSize));
417
+ clockLoop = 0; /* force immediate test end */
418
+ break;
419
+ }
420
+ blockTable[blockNb].resSize = regenSize;
421
+ }
422
+ nbLoops++;
423
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
424
+ ZSTD_freeDDict(ddict);
425
+ } else if (compressor == BMK_ZSTD_STREAM) {
426
+ ZSTD_inBuffer inBuffer;
427
+ ZSTD_outBuffer outBuffer;
428
+ ZSTD_DStream* zbd = ZSTD_createDStream();
429
+ size_t rSize;
430
+ if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure");
431
+ rSize = ZSTD_DCtx_reset(zbd, ZSTD_reset_session_only);
432
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
433
+ rSize = ZSTD_DCtx_loadDictionary(zbd, dictBuffer, dictBufferSize);
434
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_loadDictionary() failed : %s", ZSTD_getErrorName(rSize));
435
+ do {
436
+ U32 blockNb;
437
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
438
+ rSize = ZSTD_DCtx_reset(zbd, ZSTD_reset_session_only);
439
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_DCtx_reset() failed : %s", ZSTD_getErrorName(rSize));
440
+ inBuffer.src = blockTable[blockNb].cPtr;
441
+ inBuffer.size = blockTable[blockNb].cSize;
442
+ inBuffer.pos = 0;
443
+ outBuffer.dst = blockTable[blockNb].resPtr;
444
+ outBuffer.size = blockTable[blockNb].srcSize;
445
+ outBuffer.pos = 0;
446
+ rSize = ZSTD_decompressStream(zbd, &outBuffer, &inBuffer);
447
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_decompressStream() failed : %s", ZSTD_getErrorName(rSize));
448
+ blockTable[blockNb].resSize = outBuffer.pos;
449
+ }
450
+ nbLoops++;
451
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
452
+ ZSTD_freeDStream(zbd);
453
+ } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
454
+ z_stream inf;
455
+ int ret;
456
+ if (compressor == BMK_ZLIB_REUSE) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB);
457
+ else ZWRAP_setDecompressionType(ZWRAP_AUTO);
458
+ inf.zalloc = Z_NULL;
459
+ inf.zfree = Z_NULL;
460
+ inf.opaque = Z_NULL;
461
+ ret = inflateInit(&inf);
462
+ if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
463
+ do {
464
+ U32 blockNb;
465
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
466
+ if (ZWRAP_isUsingZSTDdecompression(&inf))
467
+ ret = ZWRAP_inflateReset_keepDict(&inf); /* reuse dictionary to make decompression faster; inflate will return Z_NEED_DICT only for the first time */
468
+ else
469
+ ret = inflateReset(&inf);
470
+ if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
471
+ inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
472
+ inf.avail_in = (uInt)blockTable[blockNb].cSize;
473
+ inf.total_in = 0;
474
+ inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
475
+ inf.avail_out = (uInt)blockTable[blockNb].srcSize;
476
+ inf.total_out = 0;
477
+ ret = inflate(&inf, Z_FINISH);
478
+ if (ret == Z_NEED_DICT) {
479
+ ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize);
480
+ if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
481
+ ret = inflate(&inf, Z_FINISH);
482
+ }
483
+ if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
484
+ blockTable[blockNb].resSize = inf.total_out;
485
+ }
486
+ nbLoops++;
487
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
488
+ ret = inflateEnd(&inf);
489
+ if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
490
+ } else {
491
+ z_stream inf;
492
+ if (compressor == BMK_ZLIB) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB);
493
+ else ZWRAP_setDecompressionType(ZWRAP_AUTO);
494
+ do {
495
+ U32 blockNb;
496
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
497
+ int ret;
498
+ inf.zalloc = Z_NULL;
499
+ inf.zfree = Z_NULL;
500
+ inf.opaque = Z_NULL;
501
+ ret = inflateInit(&inf);
502
+ if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
503
+ inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr;
504
+ inf.avail_in = (uInt)blockTable[blockNb].cSize;
505
+ inf.total_in = 0;
506
+ inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr;
507
+ inf.avail_out = (uInt)blockTable[blockNb].srcSize;
508
+ inf.total_out = 0;
509
+ ret = inflate(&inf, Z_FINISH);
510
+ if (ret == Z_NEED_DICT) {
511
+ ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize);
512
+ if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
513
+ ret = inflate(&inf, Z_FINISH);
514
+ }
515
+ if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
516
+ ret = inflateEnd(&inf);
517
+ if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
518
+ blockTable[blockNb].resSize = inf.total_out;
519
+ }
520
+ nbLoops++;
521
+ } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
522
+ }
523
+ { U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
524
+ if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
525
+ totalDTime += clockSpan;
526
+ dCompleted = totalDTime>maxTime;
527
+ } }
528
+
529
+ markNb = (markNb+1) % NB_MARKS;
530
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
531
+ marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio,
532
+ (double)srcSize / (double)fastestC,
533
+ (double)srcSize / (double)fastestD );
534
+
535
+ /* CRC Checking */
536
+ { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
537
+ if (crcOrig!=crcCheck) {
538
+ size_t u;
539
+ DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
540
+ for (u=0; u<srcSize; u++) {
541
+ if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
542
+ unsigned segNb, bNb, pos;
543
+ size_t bacc = 0;
544
+ DISPLAY("Decoding error at pos %u ", (unsigned)u);
545
+ for (segNb = 0; segNb < nbBlocks; segNb++) {
546
+ if (bacc + blockTable[segNb].srcSize > u) break;
547
+ bacc += blockTable[segNb].srcSize;
548
+ }
549
+ pos = (U32)(u - bacc);
550
+ bNb = pos / (128 KB);
551
+ DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
552
+ break;
553
+ }
554
+ if (u==srcSize-1) { /* should never happen */
555
+ DISPLAY("no difference detected\n");
556
+ } }
557
+ break;
558
+ } } /* CRC Checking */
559
+ #endif
560
+ } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
561
+
562
+ if (g_displayLevel == 1) {
563
+ double cSpeed = (double)srcSize / (double)fastestC;
564
+ double dSpeed = (double)srcSize / (double)fastestD;
565
+ if (g_additionalParam)
566
+ DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
567
+ else
568
+ DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
569
+ }
570
+ DISPLAYLEVEL(2, "%2i#\n", cLevel);
571
+ } /* Bench */
572
+
573
+ /* clean up */
574
+ free(blockTable);
575
+ free(compressedBuffer);
576
+ free(resultBuffer);
577
+ ZSTD_freeCCtx(ctx);
578
+ ZSTD_freeDCtx(dctx);
579
+ return 0;
580
+ }
581
+
582
+
583
+ static size_t BMK_findMaxMem(U64 requiredMem)
584
+ {
585
+ size_t const step = 64 MB;
586
+ BYTE* testmem = NULL;
587
+
588
+ requiredMem = (((requiredMem >> 26) + 1) << 26);
589
+ requiredMem += step;
590
+ if (requiredMem > maxMemory) requiredMem = maxMemory;
591
+
592
+ do {
593
+ testmem = (BYTE*)malloc((size_t)requiredMem);
594
+ requiredMem -= step;
595
+ } while (!testmem && requiredMem); /* do not allocate zero bytes */
596
+
597
+ free(testmem);
598
+ return (size_t)(requiredMem+1); /* avoid zero */
599
+ }
600
+
601
+ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
602
+ const char* displayName, int cLevel, int cLevelLast,
603
+ const size_t* fileSizes, unsigned nbFiles,
604
+ const void* dictBuffer, size_t dictBufferSize)
605
+ {
606
+ int l;
607
+
608
+ const char* pch = strrchr(displayName, '\\'); /* Windows */
609
+ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
610
+ if (pch) displayName = pch+1;
611
+
612
+ SET_REALTIME_PRIORITY;
613
+
614
+ if (g_displayLevel == 1 && !g_additionalParam)
615
+ DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
616
+ ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
617
+ (unsigned)benchedSize, g_nbIterations, (unsigned)(g_blockSize>>10));
618
+
619
+ if (cLevelLast < cLevel) cLevelLast = cLevel;
620
+
621
+ DISPLAY("benchmarking zstd %s (using ZSTD_CStream)\n", ZSTD_VERSION_STRING);
622
+ for (l=cLevel; l <= cLevelLast; l++) {
623
+ BMK_benchMem(srcBuffer, benchedSize,
624
+ displayName, l,
625
+ fileSizes, nbFiles,
626
+ dictBuffer, dictBufferSize, BMK_ZSTD_STREAM);
627
+ }
628
+
629
+ DISPLAY("benchmarking zstd %s (using ZSTD_CCtx)\n", ZSTD_VERSION_STRING);
630
+ for (l=cLevel; l <= cLevelLast; l++) {
631
+ BMK_benchMem(srcBuffer, benchedSize,
632
+ displayName, l,
633
+ fileSizes, nbFiles,
634
+ dictBuffer, dictBufferSize, BMK_ZSTD);
635
+ }
636
+
637
+ DISPLAY("benchmarking zstd %s (using zlibWrapper)\n", ZSTD_VERSION_STRING);
638
+ for (l=cLevel; l <= cLevelLast; l++) {
639
+ BMK_benchMem(srcBuffer, benchedSize,
640
+ displayName, l,
641
+ fileSizes, nbFiles,
642
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD_REUSE);
643
+ }
644
+
645
+ DISPLAY("benchmarking zstd %s (zlibWrapper not reusing a context)\n", ZSTD_VERSION_STRING);
646
+ for (l=cLevel; l <= cLevelLast; l++) {
647
+ BMK_benchMem(srcBuffer, benchedSize,
648
+ displayName, l,
649
+ fileSizes, nbFiles,
650
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD);
651
+ }
652
+
653
+
654
+ if (cLevelLast > Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION;
655
+
656
+ DISPLAY("\n");
657
+ DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION);
658
+ for (l=cLevel; l <= cLevelLast; l++) {
659
+ BMK_benchMem(srcBuffer, benchedSize,
660
+ displayName, l,
661
+ fileSizes, nbFiles,
662
+ dictBuffer, dictBufferSize, BMK_ZLIB_REUSE);
663
+ }
664
+
665
+ DISPLAY("benchmarking zlib %s (zlib not reusing a context)\n", ZLIB_VERSION);
666
+ for (l=cLevel; l <= cLevelLast; l++) {
667
+ BMK_benchMem(srcBuffer, benchedSize,
668
+ displayName, l,
669
+ fileSizes, nbFiles,
670
+ dictBuffer, dictBufferSize, BMK_ZLIB);
671
+ }
672
+
673
+ DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION);
674
+ for (l=cLevel; l <= cLevelLast; l++) {
675
+ BMK_benchMem(srcBuffer, benchedSize,
676
+ displayName, l,
677
+ fileSizes, nbFiles,
678
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE);
679
+ }
680
+
681
+ DISPLAY("benchmarking zlib %s (zlibWrapper not reusing a context)\n", ZLIB_VERSION);
682
+ for (l=cLevel; l <= cLevelLast; l++) {
683
+ BMK_benchMem(srcBuffer, benchedSize,
684
+ displayName, l,
685
+ fileSizes, nbFiles,
686
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB);
687
+ }
688
+ }
689
+
690
+
691
+ /*! BMK_loadFiles() :
692
+ Loads `buffer` with content of files listed within `fileNamesTable`.
693
+ At most, fills `buffer` entirely */
694
+ static void BMK_loadFiles(void* buffer, size_t bufferSize,
695
+ size_t* fileSizes,
696
+ const char** fileNamesTable, unsigned nbFiles)
697
+ {
698
+ size_t pos = 0, totalSize = 0;
699
+ unsigned n;
700
+ for (n=0; n<nbFiles; n++) {
701
+ FILE* f;
702
+ U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
703
+ if (UTIL_isDirectory(fileNamesTable[n])) {
704
+ DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
705
+ fileSizes[n] = 0;
706
+ continue;
707
+ }
708
+ if (fileSize == UTIL_FILESIZE_UNKNOWN) {
709
+ DISPLAYLEVEL(2, "Cannot determine size of %s ... \n", fileNamesTable[n]);
710
+ fileSizes[n] = 0;
711
+ continue;
712
+ }
713
+ f = fopen(fileNamesTable[n], "rb");
714
+ if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
715
+ DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
716
+ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
717
+ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
718
+ if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
719
+ pos += readSize; }
720
+ fileSizes[n] = (size_t)fileSize;
721
+ totalSize += (size_t)fileSize;
722
+ fclose(f);
723
+ }
724
+
725
+ if (totalSize == 0) EXM_THROW(12, "no data to bench");
726
+ }
727
+
728
+ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
729
+ const char* dictFileName, int cLevel, int cLevelLast)
730
+ {
731
+ void* srcBuffer;
732
+ size_t benchedSize;
733
+ void* dictBuffer = NULL;
734
+ size_t dictBufferSize = 0;
735
+ size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
736
+ U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
737
+ char mfName[20] = {0};
738
+
739
+ if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
740
+
741
+ /* Load dictionary */
742
+ if (dictFileName != NULL) {
743
+ U64 const dictFileSize = UTIL_getFileSize(dictFileName);
744
+ if (dictFileSize > 64 MB)
745
+ EXM_THROW(10, "dictionary file %s too large", dictFileName);
746
+ dictBufferSize = (size_t)dictFileSize;
747
+ dictBuffer = malloc(dictBufferSize);
748
+ if (dictBuffer==NULL)
749
+ EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (unsigned)dictBufferSize);
750
+ BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
751
+ }
752
+
753
+ /* Memory allocation & restrictions */
754
+ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
755
+ if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
756
+ if (benchedSize < totalSizeToLoad)
757
+ DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20));
758
+ srcBuffer = malloc(benchedSize + !benchedSize);
759
+ if (!srcBuffer) EXM_THROW(12, "not enough memory");
760
+
761
+ /* Load input buffer */
762
+ BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
763
+
764
+ /* Bench */
765
+ snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
766
+ { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
767
+ BMK_benchCLevel(srcBuffer, benchedSize,
768
+ displayName, cLevel, cLevelLast,
769
+ fileSizes, nbFiles,
770
+ dictBuffer, dictBufferSize);
771
+ }
772
+
773
+ /* clean up */
774
+ free(srcBuffer);
775
+ free(dictBuffer);
776
+ free(fileSizes);
777
+ }
778
+
779
+
780
+ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
781
+ {
782
+ char name[20] = {0};
783
+ size_t benchedSize = 10000000;
784
+ void* const srcBuffer = malloc(benchedSize);
785
+
786
+ /* Memory allocation */
787
+ if (!srcBuffer) EXM_THROW(21, "not enough memory");
788
+
789
+ /* Fill input buffer */
790
+ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
791
+
792
+ /* Bench */
793
+ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
794
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
795
+
796
+ /* clean up */
797
+ free(srcBuffer);
798
+ }
799
+
800
+
801
+ static int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
802
+ const char* dictFileName, int cLevel, int cLevelLast)
803
+ {
804
+ double const compressibility = (double)g_compressibilityDefault / 100;
805
+
806
+ if (nbFiles == 0)
807
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility);
808
+ else
809
+ BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
810
+ return 0;
811
+ }
812
+
813
+
814
+
815
+
816
+ /*-************************************
817
+ * Command Line
818
+ **************************************/
819
+ static int usage(const char* programName)
820
+ {
821
+ DISPLAY(WELCOME_MESSAGE);
822
+ DISPLAY( "Usage :\n");
823
+ DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName);
824
+ DISPLAY( "\n");
825
+ DISPLAY( "FILE : a filename\n");
826
+ DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
827
+ DISPLAY( "Arguments :\n");
828
+ DISPLAY( " -D file: use `file` as Dictionary \n");
829
+ DISPLAY( " -h/-H : display help/long help and exit\n");
830
+ DISPLAY( " -V : display Version number and exit\n");
831
+ DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
832
+ DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
833
+ #ifdef UTIL_HAS_CREATEFILELIST
834
+ DISPLAY( " -r : operate recursively on directories\n");
835
+ #endif
836
+ DISPLAY( "\n");
837
+ DISPLAY( "Benchmark arguments :\n");
838
+ DISPLAY( " -b# : benchmark file(s), using # compression level (default : %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
839
+ DISPLAY( " -e# : test all compression levels from -bX to # (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
840
+ DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
841
+ DISPLAY( " -B# : cut file into independent chunks of size # (default: no chunking)\n");
842
+ return 0;
843
+ }
844
+
845
+ static int badusage(const char* programName)
846
+ {
847
+ DISPLAYLEVEL(1, "Incorrect parameters\n");
848
+ if (g_displayLevel >= 1) usage(programName);
849
+ return 1;
850
+ }
851
+
852
+ static void waitEnter(void)
853
+ {
854
+ int unused;
855
+ DISPLAY("Press enter to continue...\n");
856
+ unused = getchar();
857
+ (void)unused;
858
+ }
859
+
860
+ /*! readU32FromChar() :
861
+ @return : unsigned integer value reach from input in `char` format
862
+ Will also modify `*stringPtr`, advancing it to position where it stopped reading.
863
+ Note : this function can overflow if digit string > MAX_UINT */
864
+ static unsigned readU32FromChar(const char** stringPtr)
865
+ {
866
+ unsigned result = 0;
867
+ while ((**stringPtr >='0') && (**stringPtr <='9'))
868
+ result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
869
+ return result;
870
+ }
871
+
872
+
873
+ #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
874
+
875
+ int main(int argCount, char** argv)
876
+ {
877
+ int argNb,
878
+ main_pause=0,
879
+ nextEntryIsDictionary=0,
880
+ operationResult=0,
881
+ nextArgumentIsFile=0;
882
+ int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
883
+ int cLevelLast = 1;
884
+ unsigned recursive = 0;
885
+ FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount);
886
+ const char* programName = argv[0];
887
+ const char* dictFileName = NULL;
888
+ char* dynNameSpace = NULL;
889
+
890
+ /* init */
891
+ if (filenames==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
892
+ displayOut = stderr;
893
+
894
+ /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
895
+ { size_t pos;
896
+ for (pos = strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } }
897
+ programName += pos;
898
+ }
899
+
900
+ /* command switches */
901
+ for(argNb=1; argNb<argCount; argNb++) {
902
+ const char* argument = argv[argNb];
903
+ if(!argument) continue; /* Protection if argument empty */
904
+
905
+ if (nextArgumentIsFile==0) {
906
+
907
+ /* long commands (--long-word) */
908
+ if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }
909
+ if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
910
+ if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage(programName)); }
911
+ if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
912
+ if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
913
+
914
+ /* Decode commands (note : aggregated commands are allowed) */
915
+ if (argument[0]=='-') {
916
+ argument++;
917
+
918
+ while (argument[0]!=0) {
919
+ switch(argument[0])
920
+ {
921
+ /* Display help */
922
+ case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */
923
+ case 'H':
924
+ case 'h': displayOut=stdout; CLEAN_RETURN(usage(programName));
925
+
926
+ /* Use file content as dictionary */
927
+ case 'D': nextEntryIsDictionary = 1; argument++; break;
928
+
929
+ /* Verbose mode */
930
+ case 'v': g_displayLevel++; argument++; break;
931
+
932
+ /* Quiet mode */
933
+ case 'q': g_displayLevel--; argument++; break;
934
+
935
+ #ifdef UTIL_HAS_CREATEFILELIST
936
+ /* recursive */
937
+ case 'r': recursive=1; argument++; break;
938
+ #endif
939
+
940
+ /* Benchmark */
941
+ case 'b':
942
+ /* first compression Level */
943
+ argument++;
944
+ cLevel = (int)readU32FromChar(&argument);
945
+ break;
946
+
947
+ /* range bench (benchmark only) */
948
+ case 'e':
949
+ /* last compression Level */
950
+ argument++;
951
+ cLevelLast = (int)readU32FromChar(&argument);
952
+ break;
953
+
954
+ /* Modify Nb Iterations (benchmark only) */
955
+ case 'i':
956
+ argument++;
957
+ { U32 const iters = readU32FromChar(&argument);
958
+ BMK_setNotificationLevel(g_displayLevel);
959
+ BMK_SetNbIterations(iters);
960
+ }
961
+ break;
962
+
963
+ /* cut input into blocks (benchmark only) */
964
+ case 'B':
965
+ argument++;
966
+ { size_t bSize = readU32FromChar(&argument);
967
+ if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
968
+ if (toupper(*argument)=='M') bSize<<=20, argument++;
969
+ if (toupper(*argument)=='B') argument++;
970
+ BMK_setNotificationLevel(g_displayLevel);
971
+ BMK_SetBlockSize(bSize);
972
+ }
973
+ break;
974
+
975
+ /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
976
+ case 'p': argument++;
977
+ if ((*argument>='0') && (*argument<='9')) {
978
+ BMK_setAdditionalParam((int)readU32FromChar(&argument));
979
+ } else
980
+ main_pause=1;
981
+ break;
982
+ /* unknown command */
983
+ default : CLEAN_RETURN(badusage(programName));
984
+ }
985
+ }
986
+ continue;
987
+ } /* if (argument[0]=='-') */
988
+
989
+ } /* if (nextArgumentIsAFile==0) */
990
+
991
+ if (nextEntryIsDictionary) {
992
+ nextEntryIsDictionary = 0;
993
+ dictFileName = argument;
994
+ continue;
995
+ }
996
+
997
+ /* add filename to list */
998
+ UTIL_refFilename(filenames, argument);
999
+ }
1000
+
1001
+ /* Welcome message (if verbose) */
1002
+ DISPLAYLEVEL(3, WELCOME_MESSAGE);
1003
+
1004
+ #ifdef UTIL_HAS_CREATEFILELIST
1005
+ if (recursive) {
1006
+ UTIL_expandFNT(&filenames, 1);
1007
+ }
1008
+ #endif
1009
+
1010
+ BMK_setNotificationLevel(g_displayLevel);
1011
+ BMK_benchFiles(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, cLevelLast);
1012
+
1013
+ _end:
1014
+ if (main_pause) waitEnter();
1015
+ free(dynNameSpace);
1016
+ UTIL_freeFileNamesTable(filenames);
1017
+ return operationResult;
1018
+ }