zstdlib 0.7.0-x86-mingw32 → 0.10.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (138) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +11 -6
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/adler32.c +0 -0
  15. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/compress.c +0 -0
  16. data/ext/zstdlib_c/zlib-1.2.12/crc32.c +1116 -0
  17. data/ext/zstdlib_c/zlib-1.2.12/crc32.h +9446 -0
  18. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.c +78 -30
  19. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.h +12 -15
  20. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzclose.c +0 -0
  21. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzguts.h +3 -2
  22. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzlib.c +5 -3
  23. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzread.c +5 -7
  24. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzwrite.c +25 -13
  25. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/infback.c +2 -1
  26. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.c +14 -14
  27. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.h +0 -0
  28. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffixed.h +0 -0
  29. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.c +39 -8
  30. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.h +3 -2
  31. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.c +3 -3
  32. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.h +0 -0
  33. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.c +27 -48
  34. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.h +0 -0
  35. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/uncompr.c +0 -0
  36. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zconf.h +0 -0
  37. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zlib.h +123 -100
  38. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.c +2 -2
  39. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.h +12 -9
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +46 -22
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.c +1 -1
  47. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +12 -19
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +2 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +41 -12
  52. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +139 -22
  53. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +47 -23
  54. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +4 -4
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +6 -5
  59. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +0 -0
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_internal.h +191 -145
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +89 -46
  68. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +27 -29
  69. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +2 -2
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress.c +2917 -868
  72. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +458 -125
  73. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +12 -11
  74. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +41 -18
  76. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +1 -1
  77. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.c +26 -298
  78. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.h +1 -1
  79. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +234 -83
  80. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +313 -138
  81. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +1 -1
  82. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +329 -150
  83. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +1 -1
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +321 -216
  87. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +9 -2
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +412 -166
  90. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +169 -453
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/huf_decompress.c +1044 -403
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +9 -9
  96. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +2 -2
  97. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +450 -105
  98. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.c +913 -273
  99. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +14 -5
  100. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +59 -12
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +699 -214
  103. data/ext/{zstdlib/zstd-1.4.5/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +2 -1
  104. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/2.4/zstdlib_c.so +0 -0
  114. data/lib/2.5/zstdlib_c.so +0 -0
  115. data/lib/2.6/zstdlib_c.so +0 -0
  116. data/lib/2.7/zstdlib_c.so +0 -0
  117. data/lib/3.0/zstdlib_c.so +0 -0
  118. data/lib/3.1/zstdlib_c.so +0 -0
  119. data/lib/zstdlib.rb +2 -2
  120. metadata +125 -116
  121. data/ext/zstdlib/zlib-1.2.11/crc32.c +0 -442
  122. data/ext/zstdlib/zlib-1.2.11/crc32.h +0 -441
  123. data/ext/zstdlib/zstd-1.4.5/lib/common/compiler.h +0 -175
  124. data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
  125. data/ext/zstdlib/zstd-1.4.5/lib/common/error_private.h +0 -80
  126. data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.c +0 -864
  127. data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.h +0 -285
  128. data/ext/zstdlib/zstd-1.4.5/lib/compress/huf_compress.c +0 -798
  129. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
  130. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
  131. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
  132. data/ext/zstdlib/zstd.mk +0 -14
  133. data/lib/2.2/zstdlib.so +0 -0
  134. data/lib/2.3/zstdlib.so +0 -0
  135. data/lib/2.4/zstdlib.so +0 -0
  136. data/lib/2.5/zstdlib.so +0 -0
  137. data/lib/2.6/zstdlib.so +0 -0
  138. data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -55,13 +55,13 @@
55
55
  /*-*******************************************************
56
56
  * Dependencies
57
57
  *********************************************************/
58
- #include <string.h> /* memcpy, memmove, memset */
59
- #include "../common/cpu.h" /* bmi2 */
58
+ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
60
59
  #include "../common/mem.h" /* low level memory routines */
61
60
  #define FSE_STATIC_LINKING_ONLY
62
61
  #include "../common/fse.h"
63
62
  #define HUF_STATIC_LINKING_ONLY
64
63
  #include "../common/huf.h"
64
+ #include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
65
65
  #include "../common/zstd_internal.h" /* blockProperties_t */
66
66
  #include "zstd_decompress_internal.h" /* ZSTD_DCtx */
67
67
  #include "zstd_ddict.h" /* ZSTD_DDictDictContent */
@@ -72,6 +72,147 @@
72
72
  #endif
73
73
 
74
74
 
75
+
76
+ /*************************************
77
+ * Multiple DDicts Hashset internals *
78
+ *************************************/
79
+
80
+ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
81
+ #define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
82
+ * Currently, that means a 0.75 load factor.
83
+ * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
84
+ * the load factor of the ddict hash set.
85
+ */
86
+
87
+ #define DDICT_HASHSET_TABLE_BASE_SIZE 64
88
+ #define DDICT_HASHSET_RESIZE_FACTOR 2
89
+
90
+ /* Hash function to determine starting position of dict insertion within the table
91
+ * Returns an index between [0, hashSet->ddictPtrTableSize]
92
+ */
93
+ static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
94
+ const U64 hash = XXH64(&dictID, sizeof(U32), 0);
95
+ /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
96
+ return hash & (hashSet->ddictPtrTableSize - 1);
97
+ }
98
+
99
+ /* Adds DDict to a hashset without resizing it.
100
+ * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
101
+ * Returns 0 if successful, or a zstd error code if something went wrong.
102
+ */
103
+ static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
104
+ const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
105
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
106
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
107
+ RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
108
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
109
+ while (hashSet->ddictPtrTable[idx] != NULL) {
110
+ /* Replace existing ddict if inserting ddict with same dictID */
111
+ if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
112
+ DEBUGLOG(4, "DictID already exists, replacing rather than adding");
113
+ hashSet->ddictPtrTable[idx] = ddict;
114
+ return 0;
115
+ }
116
+ idx &= idxRangeMask;
117
+ idx++;
118
+ }
119
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
120
+ hashSet->ddictPtrTable[idx] = ddict;
121
+ hashSet->ddictPtrCount++;
122
+ return 0;
123
+ }
124
+
125
+ /* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
126
+ * rehashes all values, allocates new table, frees old table.
127
+ * Returns 0 on success, otherwise a zstd error code.
128
+ */
129
+ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
130
+ size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
131
+ const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
132
+ const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
133
+ size_t oldTableSize = hashSet->ddictPtrTableSize;
134
+ size_t i;
135
+
136
+ DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
137
+ RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
138
+ hashSet->ddictPtrTable = newTable;
139
+ hashSet->ddictPtrTableSize = newTableSize;
140
+ hashSet->ddictPtrCount = 0;
141
+ for (i = 0; i < oldTableSize; ++i) {
142
+ if (oldTable[i] != NULL) {
143
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
144
+ }
145
+ }
146
+ ZSTD_customFree((void*)oldTable, customMem);
147
+ DEBUGLOG(4, "Finished re-hash");
148
+ return 0;
149
+ }
150
+
151
+ /* Fetches a DDict with the given dictID
152
+ * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
153
+ */
154
+ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
155
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
156
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
157
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
158
+ for (;;) {
159
+ size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
160
+ if (currDictID == dictID || currDictID == 0) {
161
+ /* currDictID == 0 implies a NULL ddict entry */
162
+ break;
163
+ } else {
164
+ idx &= idxRangeMask; /* Goes to start of table when we reach the end */
165
+ idx++;
166
+ }
167
+ }
168
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
169
+ return hashSet->ddictPtrTable[idx];
170
+ }
171
+
172
+ /* Allocates space for and returns a ddict hash set
173
+ * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
174
+ * Returns NULL if allocation failed.
175
+ */
176
+ static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
177
+ ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
178
+ DEBUGLOG(4, "Allocating new hash set");
179
+ if (!ret)
180
+ return NULL;
181
+ ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
182
+ if (!ret->ddictPtrTable) {
183
+ ZSTD_customFree(ret, customMem);
184
+ return NULL;
185
+ }
186
+ ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
187
+ ret->ddictPtrCount = 0;
188
+ return ret;
189
+ }
190
+
191
+ /* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
192
+ * Note: The ZSTD_DDict* within the table are NOT freed.
193
+ */
194
+ static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
195
+ DEBUGLOG(4, "Freeing ddict hash set");
196
+ if (hashSet && hashSet->ddictPtrTable) {
197
+ ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
198
+ }
199
+ if (hashSet) {
200
+ ZSTD_customFree(hashSet, customMem);
201
+ }
202
+ }
203
+
204
+ /* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
205
+ * Returns 0 on success, or a ZSTD error.
206
+ */
207
+ static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
208
+ DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
209
+ if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
210
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
211
+ }
212
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
213
+ return 0;
214
+ }
215
+
75
216
  /*-*************************************************************
76
217
  * Context management
77
218
  ***************************************************************/
@@ -94,11 +235,19 @@ static size_t ZSTD_startingInputLength(ZSTD_format_e format)
94
235
  return startingInputLength;
95
236
  }
96
237
 
238
+ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
239
+ {
240
+ assert(dctx->streamStage == zdss_init);
241
+ dctx->format = ZSTD_f_zstd1;
242
+ dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
243
+ dctx->outBufferMode = ZSTD_bm_buffered;
244
+ dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
245
+ dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
246
+ }
247
+
97
248
  static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
98
249
  {
99
- dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */
100
250
  dctx->staticSize = 0;
101
- dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
102
251
  dctx->ddict = NULL;
103
252
  dctx->ddictLocal = NULL;
104
253
  dctx->dictEnd = NULL;
@@ -108,12 +257,17 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
108
257
  dctx->inBuffSize = 0;
109
258
  dctx->outBuffSize = 0;
110
259
  dctx->streamStage = zdss_init;
260
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
111
261
  dctx->legacyContext = NULL;
112
262
  dctx->previousLegacyVersion = 0;
263
+ #endif
113
264
  dctx->noForwardProgress = 0;
114
265
  dctx->oversizedDuration = 0;
115
- dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
116
- dctx->outBufferMode = ZSTD_obm_buffered;
266
+ #if DYNAMIC_BMI2
267
+ dctx->bmi2 = ZSTD_cpuSupportsBmi2();
268
+ #endif
269
+ dctx->ddictSet = NULL;
270
+ ZSTD_DCtx_resetParameters(dctx);
117
271
  #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
118
272
  dctx->dictContentEndForFuzzing = NULL;
119
273
  #endif
@@ -132,11 +286,10 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
132
286
  return dctx;
133
287
  }
134
288
 
135
- ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
136
- {
137
- if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
289
+ static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
290
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
138
291
 
139
- { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem);
292
+ { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem);
140
293
  if (!dctx) return NULL;
141
294
  dctx->customMem = customMem;
142
295
  ZSTD_initDCtx_internal(dctx);
@@ -144,10 +297,15 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
144
297
  }
145
298
  }
146
299
 
300
+ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
301
+ {
302
+ return ZSTD_createDCtx_internal(customMem);
303
+ }
304
+
147
305
  ZSTD_DCtx* ZSTD_createDCtx(void)
148
306
  {
149
307
  DEBUGLOG(3, "ZSTD_createDCtx");
150
- return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
308
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
151
309
  }
152
310
 
153
311
  static void ZSTD_clearDict(ZSTD_DCtx* dctx)
@@ -164,13 +322,17 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
164
322
  RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
165
323
  { ZSTD_customMem const cMem = dctx->customMem;
166
324
  ZSTD_clearDict(dctx);
167
- ZSTD_free(dctx->inBuff, cMem);
325
+ ZSTD_customFree(dctx->inBuff, cMem);
168
326
  dctx->inBuff = NULL;
169
327
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
170
328
  if (dctx->legacyContext)
171
329
  ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
172
330
  #endif
173
- ZSTD_free(dctx, cMem);
331
+ if (dctx->ddictSet) {
332
+ ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
333
+ dctx->ddictSet = NULL;
334
+ }
335
+ ZSTD_customFree(dctx, cMem);
174
336
  return 0;
175
337
  }
176
338
  }
@@ -179,7 +341,30 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
179
341
  void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
180
342
  {
181
343
  size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
182
- memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
344
+ ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
345
+ }
346
+
347
+ /* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
348
+ * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
349
+ * accordingly sets the ddict to be used to decompress the frame.
350
+ *
351
+ * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
352
+ *
353
+ * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
354
+ */
355
+ static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
356
+ assert(dctx->refMultipleDDicts && dctx->ddictSet);
357
+ DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
358
+ if (dctx->ddict) {
359
+ const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
360
+ if (frameDDict) {
361
+ DEBUGLOG(4, "DDict found!");
362
+ ZSTD_clearDict(dctx);
363
+ dctx->dictID = dctx->fParams.dictID;
364
+ dctx->ddict = frameDDict;
365
+ dctx->dictUses = ZSTD_use_indefinitely;
366
+ }
367
+ }
183
368
  }
184
369
 
185
370
 
@@ -205,6 +390,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
205
390
  return 0;
206
391
  }
207
392
 
393
+ /*! ZSTD_isSkippableFrame() :
394
+ * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
395
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
396
+ */
397
+ unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
398
+ {
399
+ if (size < ZSTD_FRAMEIDSIZE) return 0;
400
+ { U32 const magic = MEM_readLE32(buffer);
401
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
402
+ }
403
+ return 0;
404
+ }
405
+
208
406
  /** ZSTD_frameHeaderSize_internal() :
209
407
  * srcSize must be large enough to reach header size fields.
210
408
  * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
@@ -246,7 +444,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
246
444
  const BYTE* ip = (const BYTE*)src;
247
445
  size_t const minInputSize = ZSTD_startingInputLength(format);
248
446
 
249
- memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
447
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
250
448
  if (srcSize < minInputSize) return minInputSize;
251
449
  RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
252
450
 
@@ -256,7 +454,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
256
454
  /* skippable frame */
257
455
  if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
258
456
  return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
259
- memset(zfhPtr, 0, sizeof(*zfhPtr));
457
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
260
458
  zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
261
459
  zfhPtr->frameType = ZSTD_skippableFrame;
262
460
  return 0;
@@ -291,7 +489,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
291
489
  }
292
490
  switch(dictIDSizeCode)
293
491
  {
294
- default: assert(0); /* impossible */
492
+ default:
493
+ assert(0); /* impossible */
494
+ ZSTD_FALLTHROUGH;
295
495
  case 0 : break;
296
496
  case 1 : dictID = ip[pos]; pos++; break;
297
497
  case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
@@ -299,7 +499,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
299
499
  }
300
500
  switch(fcsID)
301
501
  {
302
- default: assert(0); /* impossible */
502
+ default:
503
+ assert(0); /* impossible */
504
+ ZSTD_FALLTHROUGH;
303
505
  case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
304
506
  case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
305
507
  case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
@@ -328,7 +530,6 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
328
530
  return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
329
531
  }
330
532
 
331
-
332
533
  /** ZSTD_getFrameContentSize() :
333
534
  * compatible with legacy mode
334
535
  * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
@@ -369,6 +570,37 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
369
570
  }
370
571
  }
371
572
 
573
+ /*! ZSTD_readSkippableFrame() :
574
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
575
+ *
576
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
577
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
578
+ * in the magicVariant.
579
+ *
580
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
581
+ *
582
+ * @return : number of bytes written or a ZSTD error.
583
+ */
584
+ ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
585
+ const void* src, size_t srcSize)
586
+ {
587
+ U32 const magicNumber = MEM_readLE32(src);
588
+ size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
589
+ size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
590
+
591
+ /* check input validity */
592
+ RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
593
+ RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
594
+ RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
595
+
596
+ /* deliver payload */
597
+ if (skippableContentSize > 0 && dst != NULL)
598
+ ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
599
+ if (magicVariant != NULL)
600
+ *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
601
+ return skippableContentSize;
602
+ }
603
+
372
604
  /** ZSTD_findDecompressedSize() :
373
605
  * compatible with legacy mode
374
606
  * `srcSize` must be the exact length of some number of ZSTD compressed and/or
@@ -433,12 +665,19 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
433
665
 
434
666
  /** ZSTD_decodeFrameHeader() :
435
667
  * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
668
+ * If multiple DDict references are enabled, also will choose the correct DDict to use.
436
669
  * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
437
670
  static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
438
671
  {
439
672
  size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
440
673
  if (ZSTD_isError(result)) return result; /* invalid header */
441
674
  RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
675
+
676
+ /* Reference DDict requested by frame if dctx references multiple ddicts */
677
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
678
+ ZSTD_DCtx_selectFrameDDict(dctx);
679
+ }
680
+
442
681
  #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
443
682
  /* Skip the dictID check in fuzzing mode, because it makes the search
444
683
  * harder.
@@ -446,7 +685,9 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
446
685
  RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
447
686
  dictionary_wrong, "");
448
687
  #endif
449
- if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
688
+ dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
689
+ if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
690
+ dctx->processedCSize += headerSize;
450
691
  return 0;
451
692
  }
452
693
 
@@ -461,7 +702,7 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
461
702
  static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
462
703
  {
463
704
  ZSTD_frameSizeInfo frameSizeInfo;
464
- memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
705
+ ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
465
706
 
466
707
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
467
708
  if (ZSTD_isLegacy(src, srcSize))
@@ -516,7 +757,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
516
757
  ip += 4;
517
758
  }
518
759
 
519
- frameSizeInfo.compressedSize = ip - ipstart;
760
+ frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
520
761
  frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
521
762
  ? zfh.frameContentSize
522
763
  : nbBlocks * zfh.blockSizeMax;
@@ -569,7 +810,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
569
810
  size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
570
811
  {
571
812
  DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
572
- ZSTD_checkContinuity(dctx, blockStart);
813
+ ZSTD_checkContinuity(dctx, blockStart, blockSize);
573
814
  dctx->previousDstEnd = (const char*)blockStart + blockSize;
574
815
  return blockSize;
575
816
  }
@@ -579,12 +820,12 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
579
820
  const void* src, size_t srcSize)
580
821
  {
581
822
  DEBUGLOG(5, "ZSTD_copyRawBlock");
823
+ RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
582
824
  if (dst == NULL) {
583
825
  if (srcSize == 0) return 0;
584
826
  RETURN_ERROR(dstBuffer_null, "");
585
827
  }
586
- RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
587
- memcpy(dst, src, srcSize);
828
+ ZSTD_memcpy(dst, src, srcSize);
588
829
  return srcSize;
589
830
  }
590
831
 
@@ -592,15 +833,41 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
592
833
  BYTE b,
593
834
  size_t regenSize)
594
835
  {
836
+ RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
595
837
  if (dst == NULL) {
596
838
  if (regenSize == 0) return 0;
597
839
  RETURN_ERROR(dstBuffer_null, "");
598
840
  }
599
- RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
600
- memset(dst, b, regenSize);
841
+ ZSTD_memset(dst, b, regenSize);
601
842
  return regenSize;
602
843
  }
603
844
 
845
+ static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
846
+ {
847
+ #if ZSTD_TRACE
848
+ if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
849
+ ZSTD_Trace trace;
850
+ ZSTD_memset(&trace, 0, sizeof(trace));
851
+ trace.version = ZSTD_VERSION_NUMBER;
852
+ trace.streaming = streaming;
853
+ if (dctx->ddict) {
854
+ trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
855
+ trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
856
+ trace.dictionaryIsCold = dctx->ddictIsCold;
857
+ }
858
+ trace.uncompressedSize = (size_t)uncompressedSize;
859
+ trace.compressedSize = (size_t)compressedSize;
860
+ trace.dctx = dctx;
861
+ ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
862
+ }
863
+ #else
864
+ (void)dctx;
865
+ (void)uncompressedSize;
866
+ (void)compressedSize;
867
+ (void)streaming;
868
+ #endif
869
+ }
870
+
604
871
 
605
872
  /*! ZSTD_decompressFrame() :
606
873
  * @dctx must be properly initialized
@@ -610,8 +877,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
610
877
  void* dst, size_t dstCapacity,
611
878
  const void** srcPtr, size_t *srcSizePtr)
612
879
  {
613
- const BYTE* ip = (const BYTE*)(*srcPtr);
614
- BYTE* const ostart = (BYTE* const)dst;
880
+ const BYTE* const istart = (const BYTE*)(*srcPtr);
881
+ const BYTE* ip = istart;
882
+ BYTE* const ostart = (BYTE*)dst;
615
883
  BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
616
884
  BYTE* op = ostart;
617
885
  size_t remainingSrcSize = *srcSizePtr;
@@ -647,13 +915,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
647
915
  switch(blockProperties.blockType)
648
916
  {
649
917
  case bt_compressed:
650
- decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize, /* frame */ 1);
918
+ decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
651
919
  break;
652
920
  case bt_raw :
653
- decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
921
+ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
654
922
  break;
655
923
  case bt_rle :
656
- decodedSize = ZSTD_setRleBlock(op, oend-op, *ip, blockProperties.origSize);
924
+ decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
657
925
  break;
658
926
  case bt_reserved :
659
927
  default:
@@ -661,7 +929,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
661
929
  }
662
930
 
663
931
  if (ZSTD_isError(decodedSize)) return decodedSize;
664
- if (dctx->fParams.checksumFlag)
932
+ if (dctx->validateChecksum)
665
933
  XXH64_update(&dctx->xxhState, op, decodedSize);
666
934
  if (decodedSize != 0)
667
935
  op += decodedSize;
@@ -676,19 +944,21 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
676
944
  corruption_detected, "");
677
945
  }
678
946
  if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
679
- U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
680
- U32 checkRead;
681
947
  RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
682
- checkRead = MEM_readLE32(ip);
683
- RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
948
+ if (!dctx->forceIgnoreChecksum) {
949
+ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
950
+ U32 checkRead;
951
+ checkRead = MEM_readLE32(ip);
952
+ RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
953
+ }
684
954
  ip += 4;
685
955
  remainingSrcSize -= 4;
686
956
  }
687
-
957
+ ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
688
958
  /* Allow caller to get size read */
689
959
  *srcPtr = ip;
690
960
  *srcSizePtr = remainingSrcSize;
691
- return op-ostart;
961
+ return (size_t)(op-ostart);
692
962
  }
693
963
 
694
964
  static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
@@ -721,7 +991,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
721
991
  decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
722
992
  if (ZSTD_isError(decodedSize)) return decodedSize;
723
993
 
724
- assert(decodedSize <=- dstCapacity);
994
+ assert(decodedSize <= dstCapacity);
725
995
  dst = (BYTE*)dst + decodedSize;
726
996
  dstCapacity -= decodedSize;
727
997
 
@@ -753,7 +1023,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
753
1023
  * use this in all cases but ddict */
754
1024
  FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
755
1025
  }
756
- ZSTD_checkContinuity(dctx, dst);
1026
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
757
1027
 
758
1028
  { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
759
1029
  &src, &srcSize);
@@ -761,15 +1031,13 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
761
1031
  (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
762
1032
  && (moreThan1Frame==1),
763
1033
  srcSize_wrong,
764
- "at least one frame successfully completed, but following "
765
- "bytes are garbage: it's more likely to be a srcSize error, "
766
- "specifying more bytes than compressed size of frame(s). This "
767
- "error message replaces ERROR(prefix_unknown), which would be "
768
- "confusing, as the first header is actually correct. Note that "
769
- "one could be unlucky, it might be a corruption error instead, "
770
- "happening right at the place where we expect zstd magic "
771
- "bytes. But this is _much_ less likely than a srcSize field "
772
- "error.");
1034
+ "At least one frame successfully completed, "
1035
+ "but following bytes are garbage: "
1036
+ "it's more likely to be a srcSize error, "
1037
+ "specifying more input bytes than size of frame(s). "
1038
+ "Note: one could be unlucky, it might be a corruption error instead, "
1039
+ "happening right at the place where we expect zstd magic bytes. "
1040
+ "But this is _much_ less likely than a srcSize field error.");
773
1041
  if (ZSTD_isError(res)) return res;
774
1042
  assert(res <= dstCapacity);
775
1043
  if (res != 0)
@@ -781,7 +1049,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
781
1049
 
782
1050
  RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
783
1051
 
784
- return (BYTE*)dst - (BYTE*)dststart;
1052
+ return (size_t)((BYTE*)dst - (BYTE*)dststart);
785
1053
  }
786
1054
 
787
1055
  size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
@@ -798,7 +1066,7 @@ static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
798
1066
  switch (dctx->dictUses) {
799
1067
  default:
800
1068
  assert(0 /* Impossible */);
801
- /* fall-through */
1069
+ ZSTD_FALLTHROUGH;
802
1070
  case ZSTD_dont_use:
803
1071
  ZSTD_clearDict(dctx);
804
1072
  return NULL;
@@ -820,7 +1088,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
820
1088
  {
821
1089
  #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
822
1090
  size_t regenSize;
823
- ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1091
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem);
824
1092
  RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
825
1093
  regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
826
1094
  ZSTD_freeDCtx(dctx);
@@ -854,7 +1122,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t
854
1122
  return dctx->expected;
855
1123
  if (dctx->bType != bt_raw)
856
1124
  return dctx->expected;
857
- return MIN(MAX(inputSize, 1), dctx->expected);
1125
+ return BOUNDED(1, inputSize, dctx->expected);
858
1126
  }
859
1127
 
860
1128
  ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
@@ -862,7 +1130,9 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
862
1130
  {
863
1131
  default: /* should not happen */
864
1132
  assert(0);
1133
+ ZSTD_FALLTHROUGH;
865
1134
  case ZSTDds_getFrameHeaderSize:
1135
+ ZSTD_FALLTHROUGH;
866
1136
  case ZSTDds_decodeFrameHeader:
867
1137
  return ZSTDnit_frameHeader;
868
1138
  case ZSTDds_decodeBlockHeader:
@@ -874,6 +1144,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
874
1144
  case ZSTDds_checkChecksum:
875
1145
  return ZSTDnit_checksum;
876
1146
  case ZSTDds_decodeSkippableHeader:
1147
+ ZSTD_FALLTHROUGH;
877
1148
  case ZSTDds_skipFrame:
878
1149
  return ZSTDnit_skippableFrame;
879
1150
  }
@@ -890,7 +1161,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
890
1161
  DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
891
1162
  /* Sanity check */
892
1163
  RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
893
- if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
1164
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
1165
+
1166
+ dctx->processedCSize += srcSize;
894
1167
 
895
1168
  switch (dctx->stage)
896
1169
  {
@@ -899,21 +1172,21 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
899
1172
  if (dctx->format == ZSTD_f_zstd1) { /* allows header */
900
1173
  assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
901
1174
  if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
902
- memcpy(dctx->headerBuffer, src, srcSize);
1175
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
903
1176
  dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
904
1177
  dctx->stage = ZSTDds_decodeSkippableHeader;
905
1178
  return 0;
906
1179
  } }
907
1180
  dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
908
1181
  if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
909
- memcpy(dctx->headerBuffer, src, srcSize);
1182
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
910
1183
  dctx->expected = dctx->headerSize - srcSize;
911
1184
  dctx->stage = ZSTDds_decodeFrameHeader;
912
1185
  return 0;
913
1186
 
914
1187
  case ZSTDds_decodeFrameHeader:
915
1188
  assert(src != NULL);
916
- memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
1189
+ ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
917
1190
  FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
918
1191
  dctx->expected = ZSTD_blockHeaderSize;
919
1192
  dctx->stage = ZSTDds_decodeBlockHeader;
@@ -955,7 +1228,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
955
1228
  {
956
1229
  case bt_compressed:
957
1230
  DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
958
- rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1);
1231
+ rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
959
1232
  dctx->expected = 0; /* Streaming not supported */
960
1233
  break;
961
1234
  case bt_raw :
@@ -977,7 +1250,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
977
1250
  RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
978
1251
  DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
979
1252
  dctx->decodedSize += rSize;
980
- if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
1253
+ if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
981
1254
  dctx->previousDstEnd = (char*)dst + rSize;
982
1255
 
983
1256
  /* Stay on the same stage until we are finished streaming the block. */
@@ -995,6 +1268,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
995
1268
  dctx->expected = 4;
996
1269
  dctx->stage = ZSTDds_checkChecksum;
997
1270
  } else {
1271
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
998
1272
  dctx->expected = 0; /* ends here */
999
1273
  dctx->stage = ZSTDds_getFrameHeaderSize;
1000
1274
  }
@@ -1007,10 +1281,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
1007
1281
 
1008
1282
  case ZSTDds_checkChecksum:
1009
1283
  assert(srcSize == 4); /* guaranteed by dctx->expected */
1010
- { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
1011
- U32 const check32 = MEM_readLE32(src);
1012
- DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
1013
- RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
1284
+ {
1285
+ if (dctx->validateChecksum) {
1286
+ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
1287
+ U32 const check32 = MEM_readLE32(src);
1288
+ DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
1289
+ RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
1290
+ }
1291
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
1014
1292
  dctx->expected = 0;
1015
1293
  dctx->stage = ZSTDds_getFrameHeaderSize;
1016
1294
  return 0;
@@ -1019,7 +1297,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
1019
1297
  case ZSTDds_decodeSkippableHeader:
1020
1298
  assert(src != NULL);
1021
1299
  assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
1022
- memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
1300
+ ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
1023
1301
  dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
1024
1302
  dctx->stage = ZSTDds_skipFrame;
1025
1303
  return 0;
@@ -1075,7 +1353,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1075
1353
  workspace, workspaceSize);
1076
1354
  #else
1077
1355
  size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
1078
- dictPtr, dictEnd - dictPtr,
1356
+ dictPtr, (size_t)(dictEnd - dictPtr),
1079
1357
  workspace, workspaceSize);
1080
1358
  #endif
1081
1359
  RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
@@ -1084,40 +1362,46 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1084
1362
 
1085
1363
  { short offcodeNCount[MaxOff+1];
1086
1364
  unsigned offcodeMaxValue = MaxOff, offcodeLog;
1087
- size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
1365
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
1088
1366
  RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
1089
1367
  RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
1090
1368
  RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
1091
1369
  ZSTD_buildFSETable( entropy->OFTable,
1092
1370
  offcodeNCount, offcodeMaxValue,
1093
1371
  OF_base, OF_bits,
1094
- offcodeLog);
1372
+ offcodeLog,
1373
+ entropy->workspace, sizeof(entropy->workspace),
1374
+ /* bmi2 */0);
1095
1375
  dictPtr += offcodeHeaderSize;
1096
1376
  }
1097
1377
 
1098
1378
  { short matchlengthNCount[MaxML+1];
1099
1379
  unsigned matchlengthMaxValue = MaxML, matchlengthLog;
1100
- size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
1380
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
1101
1381
  RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
1102
1382
  RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
1103
1383
  RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
1104
1384
  ZSTD_buildFSETable( entropy->MLTable,
1105
1385
  matchlengthNCount, matchlengthMaxValue,
1106
1386
  ML_base, ML_bits,
1107
- matchlengthLog);
1387
+ matchlengthLog,
1388
+ entropy->workspace, sizeof(entropy->workspace),
1389
+ /* bmi2 */ 0);
1108
1390
  dictPtr += matchlengthHeaderSize;
1109
1391
  }
1110
1392
 
1111
1393
  { short litlengthNCount[MaxLL+1];
1112
1394
  unsigned litlengthMaxValue = MaxLL, litlengthLog;
1113
- size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
1395
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
1114
1396
  RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
1115
1397
  RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
1116
1398
  RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
1117
1399
  ZSTD_buildFSETable( entropy->LLTable,
1118
1400
  litlengthNCount, litlengthMaxValue,
1119
1401
  LL_base, LL_bits,
1120
- litlengthLog);
1402
+ litlengthLog,
1403
+ entropy->workspace, sizeof(entropy->workspace),
1404
+ /* bmi2 */ 0);
1121
1405
  dictPtr += litlengthHeaderSize;
1122
1406
  }
1123
1407
 
@@ -1131,7 +1415,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1131
1415
  entropy->rep[i] = rep;
1132
1416
  } }
1133
1417
 
1134
- return dictPtr - (const BYTE*)dict;
1418
+ return (size_t)(dictPtr - (const BYTE*)dict);
1135
1419
  }
1136
1420
 
1137
1421
  static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
@@ -1158,8 +1442,12 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
1158
1442
  size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
1159
1443
  {
1160
1444
  assert(dctx != NULL);
1445
+ #if ZSTD_TRACE
1446
+ dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
1447
+ #endif
1161
1448
  dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
1162
1449
  dctx->stage = ZSTDds_getFrameHeaderSize;
1450
+ dctx->processedCSize = 0;
1163
1451
  dctx->decodedSize = 0;
1164
1452
  dctx->previousDstEnd = NULL;
1165
1453
  dctx->prefixStart = NULL;
@@ -1170,7 +1458,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
1170
1458
  dctx->dictID = 0;
1171
1459
  dctx->bType = bt_reserved;
1172
1460
  ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
1173
- memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
1461
+ ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
1174
1462
  dctx->LLTptr = dctx->entropy.LLTable;
1175
1463
  dctx->MLTptr = dctx->entropy.MLTable;
1176
1464
  dctx->OFTptr = dctx->entropy.OFTable;
@@ -1265,7 +1553,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
1265
1553
  ZSTD_DStream* ZSTD_createDStream(void)
1266
1554
  {
1267
1555
  DEBUGLOG(3, "ZSTD_createDStream");
1268
- return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
1556
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
1269
1557
  }
1270
1558
 
1271
1559
  ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
@@ -1275,7 +1563,7 @@ ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
1275
1563
 
1276
1564
  ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
1277
1565
  {
1278
- return ZSTD_createDCtx_advanced(customMem);
1566
+ return ZSTD_createDCtx_internal(customMem);
1279
1567
  }
1280
1568
 
1281
1569
  size_t ZSTD_freeDStream(ZSTD_DStream* zds)
@@ -1373,6 +1661,16 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
1373
1661
  if (ddict) {
1374
1662
  dctx->ddict = ddict;
1375
1663
  dctx->dictUses = ZSTD_use_indefinitely;
1664
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
1665
+ if (dctx->ddictSet == NULL) {
1666
+ dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
1667
+ if (!dctx->ddictSet) {
1668
+ RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
1669
+ }
1670
+ }
1671
+ assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */
1672
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
1673
+ }
1376
1674
  }
1377
1675
  return 0;
1378
1676
  }
@@ -1394,7 +1692,7 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
1394
1692
 
1395
1693
  size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
1396
1694
  {
1397
- return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format);
1695
+ return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
1398
1696
  }
1399
1697
 
1400
1698
  ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
@@ -1411,8 +1709,16 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
1411
1709
  ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
1412
1710
  return bounds;
1413
1711
  case ZSTD_d_stableOutBuffer:
1414
- bounds.lowerBound = (int)ZSTD_obm_buffered;
1415
- bounds.upperBound = (int)ZSTD_obm_stable;
1712
+ bounds.lowerBound = (int)ZSTD_bm_buffered;
1713
+ bounds.upperBound = (int)ZSTD_bm_stable;
1714
+ return bounds;
1715
+ case ZSTD_d_forceIgnoreChecksum:
1716
+ bounds.lowerBound = (int)ZSTD_d_validateChecksum;
1717
+ bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
1718
+ return bounds;
1719
+ case ZSTD_d_refMultipleDDicts:
1720
+ bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
1721
+ bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
1416
1722
  return bounds;
1417
1723
  default:;
1418
1724
  }
@@ -1436,6 +1742,29 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
1436
1742
  RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
1437
1743
  }
1438
1744
 
1745
+ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
1746
+ {
1747
+ switch (param) {
1748
+ case ZSTD_d_windowLogMax:
1749
+ *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
1750
+ return 0;
1751
+ case ZSTD_d_format:
1752
+ *value = (int)dctx->format;
1753
+ return 0;
1754
+ case ZSTD_d_stableOutBuffer:
1755
+ *value = (int)dctx->outBufferMode;
1756
+ return 0;
1757
+ case ZSTD_d_forceIgnoreChecksum:
1758
+ *value = (int)dctx->forceIgnoreChecksum;
1759
+ return 0;
1760
+ case ZSTD_d_refMultipleDDicts:
1761
+ *value = (int)dctx->refMultipleDDicts;
1762
+ return 0;
1763
+ default:;
1764
+ }
1765
+ RETURN_ERROR(parameter_unsupported, "");
1766
+ }
1767
+
1439
1768
  size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
1440
1769
  {
1441
1770
  RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
@@ -1451,7 +1780,18 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
1451
1780
  return 0;
1452
1781
  case ZSTD_d_stableOutBuffer:
1453
1782
  CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
1454
- dctx->outBufferMode = (ZSTD_outBufferMode_e)value;
1783
+ dctx->outBufferMode = (ZSTD_bufferMode_e)value;
1784
+ return 0;
1785
+ case ZSTD_d_forceIgnoreChecksum:
1786
+ CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
1787
+ dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
1788
+ return 0;
1789
+ case ZSTD_d_refMultipleDDicts:
1790
+ CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
1791
+ if (dctx->staticSize != 0) {
1792
+ RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
1793
+ }
1794
+ dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
1455
1795
  return 0;
1456
1796
  default:;
1457
1797
  }
@@ -1469,8 +1809,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
1469
1809
  || (reset == ZSTD_reset_session_and_parameters) ) {
1470
1810
  RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1471
1811
  ZSTD_clearDict(dctx);
1472
- dctx->format = ZSTD_f_zstd1;
1473
- dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
1812
+ ZSTD_DCtx_resetParameters(dctx);
1474
1813
  }
1475
1814
  return 0;
1476
1815
  }
@@ -1484,7 +1823,8 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
1484
1823
  size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
1485
1824
  {
1486
1825
  size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
1487
- unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
1826
+ /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
1827
+ unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
1488
1828
  unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
1489
1829
  size_t const minRBSize = (size_t) neededSize;
1490
1830
  RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
@@ -1524,7 +1864,7 @@ static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const ne
1524
1864
  {
1525
1865
  if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
1526
1866
  zds->oversizedDuration++;
1527
- else
1867
+ else
1528
1868
  zds->oversizedDuration = 0;
1529
1869
  }
1530
1870
 
@@ -1538,7 +1878,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const*
1538
1878
  {
1539
1879
  ZSTD_outBuffer const expect = zds->expectedOutBuffer;
1540
1880
  /* No requirement when ZSTD_obm_stable is not enabled. */
1541
- if (zds->outBufferMode != ZSTD_obm_stable)
1881
+ if (zds->outBufferMode != ZSTD_bm_stable)
1542
1882
  return 0;
1543
1883
  /* Any buffer is allowed in zdss_init, this must be the same for every other call until
1544
1884
  * the context is reset.
@@ -1548,7 +1888,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const*
1548
1888
  /* The buffer must match our expectation exactly. */
1549
1889
  if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
1550
1890
  return 0;
1551
- RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!");
1891
+ RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
1552
1892
  }
1553
1893
 
1554
1894
  /* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
@@ -1560,7 +1900,7 @@ static size_t ZSTD_decompressContinueStream(
1560
1900
  ZSTD_DStream* zds, char** op, char* oend,
1561
1901
  void const* src, size_t srcSize) {
1562
1902
  int const isSkipFrame = ZSTD_isSkipFrame(zds);
1563
- if (zds->outBufferMode == ZSTD_obm_buffered) {
1903
+ if (zds->outBufferMode == ZSTD_bm_buffered) {
1564
1904
  size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
1565
1905
  size_t const decodedSize = ZSTD_decompressContinue(zds,
1566
1906
  zds->outBuff + zds->outStart, dstSize, src, srcSize);
@@ -1573,14 +1913,14 @@ static size_t ZSTD_decompressContinueStream(
1573
1913
  }
1574
1914
  } else {
1575
1915
  /* Write directly into the output buffer */
1576
- size_t const dstSize = isSkipFrame ? 0 : oend - *op;
1916
+ size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
1577
1917
  size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
1578
1918
  FORWARD_IF_ERROR(decodedSize, "");
1579
1919
  *op += decodedSize;
1580
1920
  /* Flushing is not needed. */
1581
1921
  zds->streamStage = zdss_read;
1582
1922
  assert(*op <= oend);
1583
- assert(zds->outBufferMode == ZSTD_obm_stable);
1923
+ assert(zds->outBufferMode == ZSTD_bm_stable);
1584
1924
  }
1585
1925
  return 0;
1586
1926
  }
@@ -1618,10 +1958,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1618
1958
  DEBUGLOG(5, "stage zdss_init => transparent reset ");
1619
1959
  zds->streamStage = zdss_loadHeader;
1620
1960
  zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
1961
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
1621
1962
  zds->legacyVersion = 0;
1963
+ #endif
1622
1964
  zds->hostageByte = 0;
1623
1965
  zds->expectedOutBuffer = *output;
1624
- /* fall-through */
1966
+ ZSTD_FALLTHROUGH;
1625
1967
 
1626
1968
  case zdss_loadHeader :
1627
1969
  DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
@@ -1635,6 +1977,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1635
1977
  } }
1636
1978
  #endif
1637
1979
  { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
1980
+ if (zds->refMultipleDDicts && zds->ddictSet) {
1981
+ ZSTD_DCtx_selectFrameDDict(zds);
1982
+ }
1638
1983
  DEBUGLOG(5, "header size : %u", (U32)hSize);
1639
1984
  if (ZSTD_isError(hSize)) {
1640
1985
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
@@ -1663,14 +2008,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1663
2008
  assert(iend >= ip);
1664
2009
  if (toLoad > remainingInput) { /* not enough input to load full header */
1665
2010
  if (remainingInput > 0) {
1666
- memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
2011
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
1667
2012
  zds->lhSize += remainingInput;
1668
2013
  }
1669
2014
  input->pos = input->size;
1670
2015
  return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
1671
2016
  }
1672
2017
  assert(ip != NULL);
1673
- memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
2018
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
1674
2019
  break;
1675
2020
  } }
1676
2021
 
@@ -1678,10 +2023,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1678
2023
  if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
1679
2024
  && zds->fParams.frameType != ZSTD_skippableFrame
1680
2025
  && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
1681
- size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
2026
+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
1682
2027
  if (cSize <= (size_t)(iend-istart)) {
1683
2028
  /* shortcut : using single-pass mode */
1684
- size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds));
2029
+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
1685
2030
  if (ZSTD_isError(decompressedSize)) return decompressedSize;
1686
2031
  DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
1687
2032
  ip = istart + cSize;
@@ -1693,7 +2038,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1693
2038
  } }
1694
2039
 
1695
2040
  /* Check output buffer is large enough for ZSTD_odm_stable. */
1696
- if (zds->outBufferMode == ZSTD_obm_stable
2041
+ if (zds->outBufferMode == ZSTD_bm_stable
1697
2042
  && zds->fParams.frameType != ZSTD_skippableFrame
1698
2043
  && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
1699
2044
  && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
@@ -1723,7 +2068,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1723
2068
 
1724
2069
  /* Adapt buffer sizes to frame header instructions */
1725
2070
  { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
1726
- size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered
2071
+ size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
1727
2072
  ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
1728
2073
  : 0;
1729
2074
 
@@ -1731,7 +2076,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1731
2076
 
1732
2077
  { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
1733
2078
  int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
1734
-
2079
+
1735
2080
  if (tooSmall || tooLarge) {
1736
2081
  size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
1737
2082
  DEBUGLOG(4, "inBuff : from %u to %u",
@@ -1745,10 +2090,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1745
2090
  bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
1746
2091
  memory_allocation, "");
1747
2092
  } else {
1748
- ZSTD_free(zds->inBuff, zds->customMem);
2093
+ ZSTD_customFree(zds->inBuff, zds->customMem);
1749
2094
  zds->inBuffSize = 0;
1750
2095
  zds->outBuffSize = 0;
1751
- zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
2096
+ zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
1752
2097
  RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
1753
2098
  }
1754
2099
  zds->inBuffSize = neededInBuffSize;
@@ -1756,11 +2101,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1756
2101
  zds->outBuffSize = neededOutBuffSize;
1757
2102
  } } }
1758
2103
  zds->streamStage = zdss_read;
1759
- /* fall-through */
2104
+ ZSTD_FALLTHROUGH;
1760
2105
 
1761
2106
  case zdss_read:
1762
2107
  DEBUGLOG(5, "stage zdss_read");
1763
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip);
2108
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
1764
2109
  DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
1765
2110
  if (neededInSize==0) { /* end of frame */
1766
2111
  zds->streamStage = zdss_init;
@@ -1775,7 +2120,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1775
2120
  } }
1776
2121
  if (ip==iend) { someMoreWork = 0; break; } /* no more input */
1777
2122
  zds->streamStage = zdss_load;
1778
- /* fall-through */
2123
+ ZSTD_FALLTHROUGH;
1779
2124
 
1780
2125
  case zdss_load:
1781
2126
  { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
@@ -1790,7 +2135,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1790
2135
  RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
1791
2136
  corruption_detected,
1792
2137
  "should never happen");
1793
- loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
2138
+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
1794
2139
  }
1795
2140
  ip += loadedSize;
1796
2141
  zds->inPos += loadedSize;
@@ -1804,7 +2149,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1804
2149
  }
1805
2150
  case zdss_flush:
1806
2151
  { size_t const toFlushSize = zds->outEnd - zds->outStart;
1807
- size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
2152
+ size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
1808
2153
  op += flushedSize;
1809
2154
  zds->outStart += flushedSize;
1810
2155
  if (flushedSize == toFlushSize) { /* flush completed */