zstdlib 0.7.0-x86-mingw32 → 0.10.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +11 -6
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/adler32.c +0 -0
  15. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/compress.c +0 -0
  16. data/ext/zstdlib_c/zlib-1.2.12/crc32.c +1116 -0
  17. data/ext/zstdlib_c/zlib-1.2.12/crc32.h +9446 -0
  18. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.c +78 -30
  19. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.h +12 -15
  20. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzclose.c +0 -0
  21. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzguts.h +3 -2
  22. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzlib.c +5 -3
  23. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzread.c +5 -7
  24. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzwrite.c +25 -13
  25. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/infback.c +2 -1
  26. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.c +14 -14
  27. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.h +0 -0
  28. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffixed.h +0 -0
  29. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.c +39 -8
  30. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.h +3 -2
  31. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.c +3 -3
  32. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.h +0 -0
  33. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.c +27 -48
  34. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.h +0 -0
  35. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/uncompr.c +0 -0
  36. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zconf.h +0 -0
  37. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zlib.h +123 -100
  38. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.c +2 -2
  39. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.h +12 -9
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +46 -22
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.c +1 -1
  47. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +12 -19
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +2 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +41 -12
  52. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +139 -22
  53. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +47 -23
  54. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +4 -4
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +6 -5
  59. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +0 -0
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_internal.h +191 -145
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +89 -46
  68. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +27 -29
  69. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +2 -2
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress.c +2917 -868
  72. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +458 -125
  73. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +12 -11
  74. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +41 -18
  76. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +1 -1
  77. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.c +26 -298
  78. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.h +1 -1
  79. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +234 -83
  80. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +313 -138
  81. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +1 -1
  82. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +329 -150
  83. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +1 -1
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +321 -216
  87. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +9 -2
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +412 -166
  90. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +169 -453
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/huf_decompress.c +1044 -403
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +9 -9
  96. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +2 -2
  97. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +450 -105
  98. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.c +913 -273
  99. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +14 -5
  100. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +59 -12
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +699 -214
  103. data/ext/{zstdlib/zstd-1.4.5/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +2 -1
  104. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/2.4/zstdlib_c.so +0 -0
  114. data/lib/2.5/zstdlib_c.so +0 -0
  115. data/lib/2.6/zstdlib_c.so +0 -0
  116. data/lib/2.7/zstdlib_c.so +0 -0
  117. data/lib/3.0/zstdlib_c.so +0 -0
  118. data/lib/3.1/zstdlib_c.so +0 -0
  119. data/lib/zstdlib.rb +2 -2
  120. metadata +125 -116
  121. data/ext/zstdlib/zlib-1.2.11/crc32.c +0 -442
  122. data/ext/zstdlib/zlib-1.2.11/crc32.h +0 -441
  123. data/ext/zstdlib/zstd-1.4.5/lib/common/compiler.h +0 -175
  124. data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
  125. data/ext/zstdlib/zstd-1.4.5/lib/common/error_private.h +0 -80
  126. data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.c +0 -864
  127. data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.h +0 -285
  128. data/ext/zstdlib/zstd-1.4.5/lib/compress/huf_compress.c +0 -798
  129. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
  130. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
  131. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
  132. data/ext/zstdlib/zstd.mk +0 -14
  133. data/lib/2.2/zstdlib.so +0 -0
  134. data/lib/2.3/zstdlib.so +0 -0
  135. data/lib/2.4/zstdlib.so +0 -0
  136. data/lib/2.5/zstdlib.so +0 -0
  137. data/lib/2.6/zstdlib.so +0 -0
  138. data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,288 +15,10 @@
15
15
 
16
16
  #include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */
17
17
  #include "hist.h" /* HIST_countFast_wksp */
18
- #include "zstd_compress_internal.h"
18
+ #include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
19
19
  #include "zstd_compress_sequences.h"
20
20
  #include "zstd_compress_literals.h"
21
21
 
22
- /*-*************************************
23
- * Superblock entropy buffer structs
24
- ***************************************/
25
- /** ZSTD_hufCTablesMetadata_t :
26
- * Stores Literals Block Type for a super-block in hType, and
27
- * huffman tree description in hufDesBuffer.
28
- * hufDesSize refers to the size of huffman tree description in bytes.
29
- * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */
30
- typedef struct {
31
- symbolEncodingType_e hType;
32
- BYTE hufDesBuffer[500]; /* TODO give name to this value */
33
- size_t hufDesSize;
34
- } ZSTD_hufCTablesMetadata_t;
35
-
36
- /** ZSTD_fseCTablesMetadata_t :
37
- * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
38
- * fse tables in fseTablesBuffer.
39
- * fseTablesSize refers to the size of fse tables in bytes.
40
- * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */
41
- typedef struct {
42
- symbolEncodingType_e llType;
43
- symbolEncodingType_e ofType;
44
- symbolEncodingType_e mlType;
45
- BYTE fseTablesBuffer[500]; /* TODO give name to this value */
46
- size_t fseTablesSize;
47
- size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */
48
- } ZSTD_fseCTablesMetadata_t;
49
-
50
- typedef struct {
51
- ZSTD_hufCTablesMetadata_t hufMetadata;
52
- ZSTD_fseCTablesMetadata_t fseMetadata;
53
- } ZSTD_entropyCTablesMetadata_t;
54
-
55
-
56
- /** ZSTD_buildSuperBlockEntropy_literal() :
57
- * Builds entropy for the super-block literals.
58
- * Stores literals block type (raw, rle, compressed, repeat) and
59
- * huffman description table to hufMetadata.
60
- * @return : size of huffman description table or error code */
61
- static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize,
62
- const ZSTD_hufCTables_t* prevHuf,
63
- ZSTD_hufCTables_t* nextHuf,
64
- ZSTD_hufCTablesMetadata_t* hufMetadata,
65
- const int disableLiteralsCompression,
66
- void* workspace, size_t wkspSize)
67
- {
68
- BYTE* const wkspStart = (BYTE*)workspace;
69
- BYTE* const wkspEnd = wkspStart + wkspSize;
70
- BYTE* const countWkspStart = wkspStart;
71
- unsigned* const countWksp = (unsigned*)workspace;
72
- const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
73
- BYTE* const nodeWksp = countWkspStart + countWkspSize;
74
- const size_t nodeWkspSize = wkspEnd-nodeWksp;
75
- unsigned maxSymbolValue = 255;
76
- unsigned huffLog = HUF_TABLELOG_DEFAULT;
77
- HUF_repeat repeat = prevHuf->repeatMode;
78
-
79
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize);
80
-
81
- /* Prepare nextEntropy assuming reusing the existing table */
82
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
83
-
84
- if (disableLiteralsCompression) {
85
- DEBUGLOG(5, "set_basic - disabled");
86
- hufMetadata->hType = set_basic;
87
- return 0;
88
- }
89
-
90
- /* small ? don't even attempt compression (speed opt) */
91
- # define COMPRESS_LITERALS_SIZE_MIN 63
92
- { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
93
- if (srcSize <= minLitSize) {
94
- DEBUGLOG(5, "set_basic - too small");
95
- hufMetadata->hType = set_basic;
96
- return 0;
97
- }
98
- }
99
-
100
- /* Scan input and build symbol stats */
101
- { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
102
- FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
103
- if (largest == srcSize) {
104
- DEBUGLOG(5, "set_rle");
105
- hufMetadata->hType = set_rle;
106
- return 0;
107
- }
108
- if (largest <= (srcSize >> 7)+4) {
109
- DEBUGLOG(5, "set_basic - no gain");
110
- hufMetadata->hType = set_basic;
111
- return 0;
112
- }
113
- }
114
-
115
- /* Validate the previous Huffman table */
116
- if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
117
- repeat = HUF_repeat_none;
118
- }
119
-
120
- /* Build Huffman Tree */
121
- memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
122
- huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
123
- { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
124
- maxSymbolValue, huffLog,
125
- nodeWksp, nodeWkspSize);
126
- FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
127
- huffLog = (U32)maxBits;
128
- { /* Build and write the CTable */
129
- size_t const newCSize = HUF_estimateCompressedSize(
130
- (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
131
- size_t const hSize = HUF_writeCTable(
132
- hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
133
- (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog);
134
- /* Check against repeating the previous CTable */
135
- if (repeat != HUF_repeat_none) {
136
- size_t const oldCSize = HUF_estimateCompressedSize(
137
- (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
138
- if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
139
- DEBUGLOG(5, "set_repeat - smaller");
140
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
141
- hufMetadata->hType = set_repeat;
142
- return 0;
143
- }
144
- }
145
- if (newCSize + hSize >= srcSize) {
146
- DEBUGLOG(5, "set_basic - no gains");
147
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
148
- hufMetadata->hType = set_basic;
149
- return 0;
150
- }
151
- DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
152
- hufMetadata->hType = set_compressed;
153
- nextHuf->repeatMode = HUF_repeat_check;
154
- return hSize;
155
- }
156
- }
157
- }
158
-
159
- /** ZSTD_buildSuperBlockEntropy_sequences() :
160
- * Builds entropy for the super-block sequences.
161
- * Stores symbol compression modes and fse table to fseMetadata.
162
- * @return : size of fse tables or error code */
163
- static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
164
- const ZSTD_fseCTables_t* prevEntropy,
165
- ZSTD_fseCTables_t* nextEntropy,
166
- const ZSTD_CCtx_params* cctxParams,
167
- ZSTD_fseCTablesMetadata_t* fseMetadata,
168
- void* workspace, size_t wkspSize)
169
- {
170
- BYTE* const wkspStart = (BYTE*)workspace;
171
- BYTE* const wkspEnd = wkspStart + wkspSize;
172
- BYTE* const countWkspStart = wkspStart;
173
- unsigned* const countWksp = (unsigned*)workspace;
174
- const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned);
175
- BYTE* const cTableWksp = countWkspStart + countWkspSize;
176
- const size_t cTableWkspSize = wkspEnd-cTableWksp;
177
- ZSTD_strategy const strategy = cctxParams->cParams.strategy;
178
- FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
179
- FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
180
- FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
181
- const BYTE* const ofCodeTable = seqStorePtr->ofCode;
182
- const BYTE* const llCodeTable = seqStorePtr->llCode;
183
- const BYTE* const mlCodeTable = seqStorePtr->mlCode;
184
- size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
185
- BYTE* const ostart = fseMetadata->fseTablesBuffer;
186
- BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
187
- BYTE* op = ostart;
188
-
189
- assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE));
190
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq);
191
- memset(workspace, 0, wkspSize);
192
-
193
- fseMetadata->lastCountSize = 0;
194
- /* convert length/distances into codes */
195
- ZSTD_seqToCodes(seqStorePtr);
196
- /* build CTable for Literal Lengths */
197
- { U32 LLtype;
198
- unsigned max = MaxLL;
199
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
200
- DEBUGLOG(5, "Building LL table");
201
- nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
202
- LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
203
- countWksp, max, mostFrequent, nbSeq,
204
- LLFSELog, prevEntropy->litlengthCTable,
205
- LL_defaultNorm, LL_defaultNormLog,
206
- ZSTD_defaultAllowed, strategy);
207
- assert(set_basic < set_compressed && set_rle < set_compressed);
208
- assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
209
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
210
- countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
211
- prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable),
212
- cTableWksp, cTableWkspSize);
213
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
214
- if (LLtype == set_compressed)
215
- fseMetadata->lastCountSize = countSize;
216
- op += countSize;
217
- fseMetadata->llType = (symbolEncodingType_e) LLtype;
218
- } }
219
- /* build CTable for Offsets */
220
- { U32 Offtype;
221
- unsigned max = MaxOff;
222
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
223
- /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
224
- ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
225
- DEBUGLOG(5, "Building OF table");
226
- nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
227
- Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
228
- countWksp, max, mostFrequent, nbSeq,
229
- OffFSELog, prevEntropy->offcodeCTable,
230
- OF_defaultNorm, OF_defaultNormLog,
231
- defaultPolicy, strategy);
232
- assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
233
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
234
- countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
235
- prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable),
236
- cTableWksp, cTableWkspSize);
237
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
238
- if (Offtype == set_compressed)
239
- fseMetadata->lastCountSize = countSize;
240
- op += countSize;
241
- fseMetadata->ofType = (symbolEncodingType_e) Offtype;
242
- } }
243
- /* build CTable for MatchLengths */
244
- { U32 MLtype;
245
- unsigned max = MaxML;
246
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
247
- DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
248
- nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
249
- MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
250
- countWksp, max, mostFrequent, nbSeq,
251
- MLFSELog, prevEntropy->matchlengthCTable,
252
- ML_defaultNorm, ML_defaultNormLog,
253
- ZSTD_defaultAllowed, strategy);
254
- assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
255
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
256
- countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
257
- prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable),
258
- cTableWksp, cTableWkspSize);
259
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
260
- if (MLtype == set_compressed)
261
- fseMetadata->lastCountSize = countSize;
262
- op += countSize;
263
- fseMetadata->mlType = (symbolEncodingType_e) MLtype;
264
- } }
265
- assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer));
266
- return op-ostart;
267
- }
268
-
269
-
270
- /** ZSTD_buildSuperBlockEntropy() :
271
- * Builds entropy for the super-block.
272
- * @return : 0 on success or error code */
273
- static size_t
274
- ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
275
- const ZSTD_entropyCTables_t* prevEntropy,
276
- ZSTD_entropyCTables_t* nextEntropy,
277
- const ZSTD_CCtx_params* cctxParams,
278
- ZSTD_entropyCTablesMetadata_t* entropyMetadata,
279
- void* workspace, size_t wkspSize)
280
- {
281
- size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
282
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy");
283
- entropyMetadata->hufMetadata.hufDesSize =
284
- ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize,
285
- &prevEntropy->huf, &nextEntropy->huf,
286
- &entropyMetadata->hufMetadata,
287
- ZSTD_disableLiteralsCompression(cctxParams),
288
- workspace, wkspSize);
289
- FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed");
290
- entropyMetadata->fseMetadata.fseTablesSize =
291
- ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr,
292
- &prevEntropy->fse, &nextEntropy->fse,
293
- cctxParams,
294
- &entropyMetadata->fseMetadata,
295
- workspace, wkspSize);
296
- FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed");
297
- return 0;
298
- }
299
-
300
22
  /** ZSTD_compressSubBlock_literal() :
301
23
  * Compresses literals section for a sub-block.
302
24
  * When we have to write the Huffman table we will sometimes choose a header
@@ -304,7 +26,7 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
304
26
  * before we know the table size + compressed size, so we have a bound on the
305
27
  * table size. If we guessed incorrectly, we fall back to uncompressed literals.
306
28
  *
307
- * We write the header when writeEntropy=1 and set entropyWrriten=1 when we succeeded
29
+ * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
308
30
  * in writing the header, otherwise it is set to 0.
309
31
  *
310
32
  * hufMetadata->hType has literals block type info.
@@ -348,7 +70,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
348
70
  assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
349
71
 
350
72
  if (writeEntropy && hufMetadata->hType == set_compressed) {
351
- memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
73
+ ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
352
74
  op += hufMetadata->hufDesSize;
353
75
  cLitSize += hufMetadata->hufDesSize;
354
76
  DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
@@ -410,6 +132,7 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef*
410
132
  const seqDef* sp = sstart;
411
133
  size_t matchLengthSum = 0;
412
134
  size_t litLengthSum = 0;
135
+ (void)(litLengthSum); /* suppress unused variable warning on some environments */
413
136
  while (send-sp > 0) {
414
137
  ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
415
138
  litLengthSum += seqLen.litLength;
@@ -474,7 +197,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
474
197
  const U32 MLtype = fseMetadata->mlType;
475
198
  DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
476
199
  *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
477
- memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
200
+ ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
478
201
  op += fseMetadata->fseTablesSize;
479
202
  } else {
480
203
  const U32 repeat = set_repeat;
@@ -602,8 +325,8 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit
602
325
  static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
603
326
  const BYTE* codeTable, unsigned maxCode,
604
327
  size_t nbSeq, const FSE_CTable* fseCTable,
605
- const U32* additionalBits,
606
- short const* defaultNorm, U32 defaultNormLog,
328
+ const U8* additionalBits,
329
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
607
330
  void* workspace, size_t wkspSize)
608
331
  {
609
332
  unsigned* const countWksp = (unsigned*)workspace;
@@ -615,7 +338,11 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
615
338
 
616
339
  HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
617
340
  if (type == set_basic) {
618
- cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max);
341
+ /* We selected this encoding type, so it must be valid. */
342
+ assert(max <= defaultMax);
343
+ cSymbolTypeSizeEstimateInBits = max <= defaultMax
344
+ ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
345
+ : ERROR(GENERIC);
619
346
  } else if (type == set_rle) {
620
347
  cSymbolTypeSizeEstimateInBits = 0;
621
348
  } else if (type == set_compressed || type == set_repeat) {
@@ -639,19 +366,20 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
639
366
  void* workspace, size_t wkspSize,
640
367
  int writeEntropy)
641
368
  {
642
- size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
369
+ size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
643
370
  size_t cSeqSizeEstimate = 0;
371
+ if (nbSeq == 0) return sequencesSectionHeaderSize;
644
372
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
645
373
  nbSeq, fseTables->offcodeCTable, NULL,
646
- OF_defaultNorm, OF_defaultNormLog,
374
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
647
375
  workspace, wkspSize);
648
376
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
649
377
  nbSeq, fseTables->litlengthCTable, LL_bits,
650
- LL_defaultNorm, LL_defaultNormLog,
378
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
651
379
  workspace, wkspSize);
652
380
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
653
381
  nbSeq, fseTables->matchlengthCTable, ML_bits,
654
- ML_defaultNorm, ML_defaultNormLog,
382
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
655
383
  workspace, wkspSize);
656
384
  if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
657
385
  return cSeqSizeEstimate + sequencesSectionHeaderSize;
@@ -747,7 +475,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
747
475
  /* I think there is an optimization opportunity here.
748
476
  * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
749
477
  * since it recalculates estimate from scratch.
750
- * For example, it would recount literal distribution and symbol codes everytime.
478
+ * For example, it would recount literal distribution and symbol codes every time.
751
479
  */
752
480
  cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
753
481
  &nextCBlock->entropy, entropyMetadata,
@@ -790,7 +518,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
790
518
  } while (!lastSequence);
791
519
  if (writeLitEntropy) {
792
520
  DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
793
- memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
521
+ ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
794
522
  }
795
523
  if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
796
524
  /* If we haven't written our entropy tables, then we've violated our contract and
@@ -809,11 +537,11 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
809
537
  if (sp < send) {
810
538
  seqDef const* seq;
811
539
  repcodes_t rep;
812
- memcpy(&rep, prevCBlock->rep, sizeof(rep));
540
+ ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
813
541
  for (seq = sstart; seq < sp; ++seq) {
814
- rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
542
+ ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
815
543
  }
816
- memcpy(nextCBlock->rep, &rep, sizeof(rep));
544
+ ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
817
545
  }
818
546
  }
819
547
  DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
@@ -826,12 +554,12 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
826
554
  unsigned lastBlock) {
827
555
  ZSTD_entropyCTablesMetadata_t entropyMetadata;
828
556
 
829
- FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore,
557
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
830
558
  &zc->blockState.prevCBlock->entropy,
831
559
  &zc->blockState.nextCBlock->entropy,
832
560
  &zc->appliedParams,
833
561
  &entropyMetadata,
834
- zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
562
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
835
563
 
836
564
  return ZSTD_compressSubBlock_multi(&zc->seqStore,
837
565
  zc->blockState.prevCBlock,
@@ -841,5 +569,5 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
841
569
  dst, dstCapacity,
842
570
  src, srcSize,
843
571
  zc->bmi2, lastBlock,
844
- zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
572
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
845
573
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the