zstdlib 0.7.0-x86-mingw32 → 0.10.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +11 -6
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/adler32.c +0 -0
  15. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/compress.c +0 -0
  16. data/ext/zstdlib_c/zlib-1.2.12/crc32.c +1116 -0
  17. data/ext/zstdlib_c/zlib-1.2.12/crc32.h +9446 -0
  18. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.c +78 -30
  19. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/deflate.h +12 -15
  20. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzclose.c +0 -0
  21. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzguts.h +3 -2
  22. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzlib.c +5 -3
  23. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzread.c +5 -7
  24. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/gzwrite.c +25 -13
  25. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/infback.c +2 -1
  26. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.c +14 -14
  27. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffast.h +0 -0
  28. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inffixed.h +0 -0
  29. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.c +39 -8
  30. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inflate.h +3 -2
  31. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.c +3 -3
  32. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/inftrees.h +0 -0
  33. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.c +27 -48
  34. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/trees.h +0 -0
  35. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/uncompr.c +0 -0
  36. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zconf.h +0 -0
  37. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zlib.h +123 -100
  38. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.c +2 -2
  39. data/ext/{zstdlib/zlib-1.2.11 → zstdlib_c/zlib-1.2.12}/zutil.h +12 -9
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +46 -22
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.c +1 -1
  47. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +12 -19
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +2 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +41 -12
  52. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +139 -22
  53. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +47 -23
  54. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +4 -4
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +6 -5
  59. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +0 -0
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_internal.h +191 -145
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +89 -46
  68. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +27 -29
  69. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +2 -2
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress.c +2917 -868
  72. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +458 -125
  73. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +12 -11
  74. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +41 -18
  76. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +1 -1
  77. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.c +26 -298
  78. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_superblock.h +1 -1
  79. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +234 -83
  80. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +313 -138
  81. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +1 -1
  82. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +329 -150
  83. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +1 -1
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +321 -216
  87. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +9 -2
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +412 -166
  90. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +169 -453
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/huf_decompress.c +1044 -403
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +9 -9
  96. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +2 -2
  97. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +450 -105
  98. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.c +913 -273
  99. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +14 -5
  100. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +59 -12
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +699 -214
  103. data/ext/{zstdlib/zstd-1.4.5/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +2 -1
  104. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.5 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/2.4/zstdlib_c.so +0 -0
  114. data/lib/2.5/zstdlib_c.so +0 -0
  115. data/lib/2.6/zstdlib_c.so +0 -0
  116. data/lib/2.7/zstdlib_c.so +0 -0
  117. data/lib/3.0/zstdlib_c.so +0 -0
  118. data/lib/3.1/zstdlib_c.so +0 -0
  119. data/lib/zstdlib.rb +2 -2
  120. metadata +125 -116
  121. data/ext/zstdlib/zlib-1.2.11/crc32.c +0 -442
  122. data/ext/zstdlib/zlib-1.2.11/crc32.h +0 -441
  123. data/ext/zstdlib/zstd-1.4.5/lib/common/compiler.h +0 -175
  124. data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
  125. data/ext/zstdlib/zstd-1.4.5/lib/common/error_private.h +0 -80
  126. data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.c +0 -864
  127. data/ext/zstdlib/zstd-1.4.5/lib/common/xxhash.h +0 -285
  128. data/ext/zstdlib/zstd-1.4.5/lib/compress/huf_compress.c +0 -798
  129. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
  130. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
  131. data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
  132. data/ext/zstdlib/zstd.mk +0 -14
  133. data/lib/2.2/zstdlib.so +0 -0
  134. data/lib/2.3/zstdlib.so +0 -0
  135. data/lib/2.4/zstdlib.so +0 -0
  136. data/lib/2.5/zstdlib.so +0 -0
  137. data/lib/2.6/zstdlib.so +0 -0
  138. data/lib/2.7/zstdlib.so +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -55,13 +55,13 @@
55
55
  /*-*******************************************************
56
56
  * Dependencies
57
57
  *********************************************************/
58
- #include <string.h> /* memcpy, memmove, memset */
59
- #include "../common/cpu.h" /* bmi2 */
58
+ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
60
59
  #include "../common/mem.h" /* low level memory routines */
61
60
  #define FSE_STATIC_LINKING_ONLY
62
61
  #include "../common/fse.h"
63
62
  #define HUF_STATIC_LINKING_ONLY
64
63
  #include "../common/huf.h"
64
+ #include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
65
65
  #include "../common/zstd_internal.h" /* blockProperties_t */
66
66
  #include "zstd_decompress_internal.h" /* ZSTD_DCtx */
67
67
  #include "zstd_ddict.h" /* ZSTD_DDictDictContent */
@@ -72,6 +72,147 @@
72
72
  #endif
73
73
 
74
74
 
75
+
76
+ /*************************************
77
+ * Multiple DDicts Hashset internals *
78
+ *************************************/
79
+
80
+ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
81
+ #define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
82
+ * Currently, that means a 0.75 load factor.
83
+ * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
84
+ * the load factor of the ddict hash set.
85
+ */
86
+
87
+ #define DDICT_HASHSET_TABLE_BASE_SIZE 64
88
+ #define DDICT_HASHSET_RESIZE_FACTOR 2
89
+
90
+ /* Hash function to determine starting position of dict insertion within the table
91
+ * Returns an index between [0, hashSet->ddictPtrTableSize]
92
+ */
93
+ static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
94
+ const U64 hash = XXH64(&dictID, sizeof(U32), 0);
95
+ /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
96
+ return hash & (hashSet->ddictPtrTableSize - 1);
97
+ }
98
+
99
+ /* Adds DDict to a hashset without resizing it.
100
+ * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
101
+ * Returns 0 if successful, or a zstd error code if something went wrong.
102
+ */
103
+ static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
104
+ const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
105
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
106
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
107
+ RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
108
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
109
+ while (hashSet->ddictPtrTable[idx] != NULL) {
110
+ /* Replace existing ddict if inserting ddict with same dictID */
111
+ if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
112
+ DEBUGLOG(4, "DictID already exists, replacing rather than adding");
113
+ hashSet->ddictPtrTable[idx] = ddict;
114
+ return 0;
115
+ }
116
+ idx &= idxRangeMask;
117
+ idx++;
118
+ }
119
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
120
+ hashSet->ddictPtrTable[idx] = ddict;
121
+ hashSet->ddictPtrCount++;
122
+ return 0;
123
+ }
124
+
125
+ /* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
126
+ * rehashes all values, allocates new table, frees old table.
127
+ * Returns 0 on success, otherwise a zstd error code.
128
+ */
129
+ static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
130
+ size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
131
+ const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem);
132
+ const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
133
+ size_t oldTableSize = hashSet->ddictPtrTableSize;
134
+ size_t i;
135
+
136
+ DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
137
+ RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
138
+ hashSet->ddictPtrTable = newTable;
139
+ hashSet->ddictPtrTableSize = newTableSize;
140
+ hashSet->ddictPtrCount = 0;
141
+ for (i = 0; i < oldTableSize; ++i) {
142
+ if (oldTable[i] != NULL) {
143
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
144
+ }
145
+ }
146
+ ZSTD_customFree((void*)oldTable, customMem);
147
+ DEBUGLOG(4, "Finished re-hash");
148
+ return 0;
149
+ }
150
+
151
+ /* Fetches a DDict with the given dictID
152
+ * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
153
+ */
154
+ static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
155
+ size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
156
+ const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
157
+ DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
158
+ for (;;) {
159
+ size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
160
+ if (currDictID == dictID || currDictID == 0) {
161
+ /* currDictID == 0 implies a NULL ddict entry */
162
+ break;
163
+ } else {
164
+ idx &= idxRangeMask; /* Goes to start of table when we reach the end */
165
+ idx++;
166
+ }
167
+ }
168
+ DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
169
+ return hashSet->ddictPtrTable[idx];
170
+ }
171
+
172
+ /* Allocates space for and returns a ddict hash set
173
+ * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
174
+ * Returns NULL if allocation failed.
175
+ */
176
+ static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
177
+ ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
178
+ DEBUGLOG(4, "Allocating new hash set");
179
+ if (!ret)
180
+ return NULL;
181
+ ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem);
182
+ if (!ret->ddictPtrTable) {
183
+ ZSTD_customFree(ret, customMem);
184
+ return NULL;
185
+ }
186
+ ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
187
+ ret->ddictPtrCount = 0;
188
+ return ret;
189
+ }
190
+
191
+ /* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
192
+ * Note: The ZSTD_DDict* within the table are NOT freed.
193
+ */
194
+ static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
195
+ DEBUGLOG(4, "Freeing ddict hash set");
196
+ if (hashSet && hashSet->ddictPtrTable) {
197
+ ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
198
+ }
199
+ if (hashSet) {
200
+ ZSTD_customFree(hashSet, customMem);
201
+ }
202
+ }
203
+
204
+ /* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
205
+ * Returns 0 on success, or a ZSTD error.
206
+ */
207
+ static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
208
+ DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
209
+ if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
210
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
211
+ }
212
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
213
+ return 0;
214
+ }
215
+
75
216
  /*-*************************************************************
76
217
  * Context management
77
218
  ***************************************************************/
@@ -94,11 +235,19 @@ static size_t ZSTD_startingInputLength(ZSTD_format_e format)
94
235
  return startingInputLength;
95
236
  }
96
237
 
238
+ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
239
+ {
240
+ assert(dctx->streamStage == zdss_init);
241
+ dctx->format = ZSTD_f_zstd1;
242
+ dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
243
+ dctx->outBufferMode = ZSTD_bm_buffered;
244
+ dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
245
+ dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
246
+ }
247
+
97
248
  static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
98
249
  {
99
- dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */
100
250
  dctx->staticSize = 0;
101
- dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
102
251
  dctx->ddict = NULL;
103
252
  dctx->ddictLocal = NULL;
104
253
  dctx->dictEnd = NULL;
@@ -108,12 +257,17 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
108
257
  dctx->inBuffSize = 0;
109
258
  dctx->outBuffSize = 0;
110
259
  dctx->streamStage = zdss_init;
260
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
111
261
  dctx->legacyContext = NULL;
112
262
  dctx->previousLegacyVersion = 0;
263
+ #endif
113
264
  dctx->noForwardProgress = 0;
114
265
  dctx->oversizedDuration = 0;
115
- dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
116
- dctx->outBufferMode = ZSTD_obm_buffered;
266
+ #if DYNAMIC_BMI2
267
+ dctx->bmi2 = ZSTD_cpuSupportsBmi2();
268
+ #endif
269
+ dctx->ddictSet = NULL;
270
+ ZSTD_DCtx_resetParameters(dctx);
117
271
  #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
118
272
  dctx->dictContentEndForFuzzing = NULL;
119
273
  #endif
@@ -132,11 +286,10 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
132
286
  return dctx;
133
287
  }
134
288
 
135
- ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
136
- {
137
- if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
289
+ static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
290
+ if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
138
291
 
139
- { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem);
292
+ { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem);
140
293
  if (!dctx) return NULL;
141
294
  dctx->customMem = customMem;
142
295
  ZSTD_initDCtx_internal(dctx);
@@ -144,10 +297,15 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
144
297
  }
145
298
  }
146
299
 
300
+ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
301
+ {
302
+ return ZSTD_createDCtx_internal(customMem);
303
+ }
304
+
147
305
  ZSTD_DCtx* ZSTD_createDCtx(void)
148
306
  {
149
307
  DEBUGLOG(3, "ZSTD_createDCtx");
150
- return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
308
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
151
309
  }
152
310
 
153
311
  static void ZSTD_clearDict(ZSTD_DCtx* dctx)
@@ -164,13 +322,17 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
164
322
  RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
165
323
  { ZSTD_customMem const cMem = dctx->customMem;
166
324
  ZSTD_clearDict(dctx);
167
- ZSTD_free(dctx->inBuff, cMem);
325
+ ZSTD_customFree(dctx->inBuff, cMem);
168
326
  dctx->inBuff = NULL;
169
327
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
170
328
  if (dctx->legacyContext)
171
329
  ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
172
330
  #endif
173
- ZSTD_free(dctx, cMem);
331
+ if (dctx->ddictSet) {
332
+ ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
333
+ dctx->ddictSet = NULL;
334
+ }
335
+ ZSTD_customFree(dctx, cMem);
174
336
  return 0;
175
337
  }
176
338
  }
@@ -179,7 +341,30 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
179
341
  void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
180
342
  {
181
343
  size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
182
- memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
344
+ ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
345
+ }
346
+
347
+ /* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
348
+ * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
349
+ * accordingly sets the ddict to be used to decompress the frame.
350
+ *
351
+ * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
352
+ *
353
+ * ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
354
+ */
355
+ static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
356
+ assert(dctx->refMultipleDDicts && dctx->ddictSet);
357
+ DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
358
+ if (dctx->ddict) {
359
+ const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
360
+ if (frameDDict) {
361
+ DEBUGLOG(4, "DDict found!");
362
+ ZSTD_clearDict(dctx);
363
+ dctx->dictID = dctx->fParams.dictID;
364
+ dctx->ddict = frameDDict;
365
+ dctx->dictUses = ZSTD_use_indefinitely;
366
+ }
367
+ }
183
368
  }
184
369
 
185
370
 
@@ -205,6 +390,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
205
390
  return 0;
206
391
  }
207
392
 
393
+ /*! ZSTD_isSkippableFrame() :
394
+ * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
395
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
396
+ */
397
+ unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
398
+ {
399
+ if (size < ZSTD_FRAMEIDSIZE) return 0;
400
+ { U32 const magic = MEM_readLE32(buffer);
401
+ if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
402
+ }
403
+ return 0;
404
+ }
405
+
208
406
  /** ZSTD_frameHeaderSize_internal() :
209
407
  * srcSize must be large enough to reach header size fields.
210
408
  * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
@@ -246,7 +444,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
246
444
  const BYTE* ip = (const BYTE*)src;
247
445
  size_t const minInputSize = ZSTD_startingInputLength(format);
248
446
 
249
- memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
447
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
250
448
  if (srcSize < minInputSize) return minInputSize;
251
449
  RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
252
450
 
@@ -256,7 +454,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
256
454
  /* skippable frame */
257
455
  if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
258
456
  return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
259
- memset(zfhPtr, 0, sizeof(*zfhPtr));
457
+ ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
260
458
  zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
261
459
  zfhPtr->frameType = ZSTD_skippableFrame;
262
460
  return 0;
@@ -291,7 +489,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
291
489
  }
292
490
  switch(dictIDSizeCode)
293
491
  {
294
- default: assert(0); /* impossible */
492
+ default:
493
+ assert(0); /* impossible */
494
+ ZSTD_FALLTHROUGH;
295
495
  case 0 : break;
296
496
  case 1 : dictID = ip[pos]; pos++; break;
297
497
  case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
@@ -299,7 +499,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
299
499
  }
300
500
  switch(fcsID)
301
501
  {
302
- default: assert(0); /* impossible */
502
+ default:
503
+ assert(0); /* impossible */
504
+ ZSTD_FALLTHROUGH;
303
505
  case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
304
506
  case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
305
507
  case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
@@ -328,7 +530,6 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
328
530
  return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
329
531
  }
330
532
 
331
-
332
533
  /** ZSTD_getFrameContentSize() :
333
534
  * compatible with legacy mode
334
535
  * @return : decompressed size of the single frame pointed to be `src` if known, otherwise
@@ -369,6 +570,37 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
369
570
  }
370
571
  }
371
572
 
573
+ /*! ZSTD_readSkippableFrame() :
574
+ * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
575
+ *
576
+ * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
577
+ * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
578
+ * in the magicVariant.
579
+ *
580
+ * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
581
+ *
582
+ * @return : number of bytes written or a ZSTD error.
583
+ */
584
+ ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
585
+ const void* src, size_t srcSize)
586
+ {
587
+ U32 const magicNumber = MEM_readLE32(src);
588
+ size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
589
+ size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
590
+
591
+ /* check input validity */
592
+ RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
593
+ RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
594
+ RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
595
+
596
+ /* deliver payload */
597
+ if (skippableContentSize > 0 && dst != NULL)
598
+ ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
599
+ if (magicVariant != NULL)
600
+ *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
601
+ return skippableContentSize;
602
+ }
603
+
372
604
  /** ZSTD_findDecompressedSize() :
373
605
  * compatible with legacy mode
374
606
  * `srcSize` must be the exact length of some number of ZSTD compressed and/or
@@ -433,12 +665,19 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
433
665
 
434
666
  /** ZSTD_decodeFrameHeader() :
435
667
  * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
668
+ * If multiple DDict references are enabled, also will choose the correct DDict to use.
436
669
  * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
437
670
  static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
438
671
  {
439
672
  size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
440
673
  if (ZSTD_isError(result)) return result; /* invalid header */
441
674
  RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
675
+
676
+ /* Reference DDict requested by frame if dctx references multiple ddicts */
677
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
678
+ ZSTD_DCtx_selectFrameDDict(dctx);
679
+ }
680
+
442
681
  #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
443
682
  /* Skip the dictID check in fuzzing mode, because it makes the search
444
683
  * harder.
@@ -446,7 +685,9 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
446
685
  RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
447
686
  dictionary_wrong, "");
448
687
  #endif
449
- if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
688
+ dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
689
+ if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
690
+ dctx->processedCSize += headerSize;
450
691
  return 0;
451
692
  }
452
693
 
@@ -461,7 +702,7 @@ static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
461
702
  static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
462
703
  {
463
704
  ZSTD_frameSizeInfo frameSizeInfo;
464
- memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
705
+ ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
465
706
 
466
707
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
467
708
  if (ZSTD_isLegacy(src, srcSize))
@@ -516,7 +757,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
516
757
  ip += 4;
517
758
  }
518
759
 
519
- frameSizeInfo.compressedSize = ip - ipstart;
760
+ frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
520
761
  frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
521
762
  ? zfh.frameContentSize
522
763
  : nbBlocks * zfh.blockSizeMax;
@@ -569,7 +810,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
569
810
  size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
570
811
  {
571
812
  DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
572
- ZSTD_checkContinuity(dctx, blockStart);
813
+ ZSTD_checkContinuity(dctx, blockStart, blockSize);
573
814
  dctx->previousDstEnd = (const char*)blockStart + blockSize;
574
815
  return blockSize;
575
816
  }
@@ -579,12 +820,12 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
579
820
  const void* src, size_t srcSize)
580
821
  {
581
822
  DEBUGLOG(5, "ZSTD_copyRawBlock");
823
+ RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
582
824
  if (dst == NULL) {
583
825
  if (srcSize == 0) return 0;
584
826
  RETURN_ERROR(dstBuffer_null, "");
585
827
  }
586
- RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
587
- memcpy(dst, src, srcSize);
828
+ ZSTD_memcpy(dst, src, srcSize);
588
829
  return srcSize;
589
830
  }
590
831
 
@@ -592,15 +833,41 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
592
833
  BYTE b,
593
834
  size_t regenSize)
594
835
  {
836
+ RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
595
837
  if (dst == NULL) {
596
838
  if (regenSize == 0) return 0;
597
839
  RETURN_ERROR(dstBuffer_null, "");
598
840
  }
599
- RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
600
- memset(dst, b, regenSize);
841
+ ZSTD_memset(dst, b, regenSize);
601
842
  return regenSize;
602
843
  }
603
844
 
845
+ static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
846
+ {
847
+ #if ZSTD_TRACE
848
+ if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
849
+ ZSTD_Trace trace;
850
+ ZSTD_memset(&trace, 0, sizeof(trace));
851
+ trace.version = ZSTD_VERSION_NUMBER;
852
+ trace.streaming = streaming;
853
+ if (dctx->ddict) {
854
+ trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
855
+ trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
856
+ trace.dictionaryIsCold = dctx->ddictIsCold;
857
+ }
858
+ trace.uncompressedSize = (size_t)uncompressedSize;
859
+ trace.compressedSize = (size_t)compressedSize;
860
+ trace.dctx = dctx;
861
+ ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
862
+ }
863
+ #else
864
+ (void)dctx;
865
+ (void)uncompressedSize;
866
+ (void)compressedSize;
867
+ (void)streaming;
868
+ #endif
869
+ }
870
+
604
871
 
605
872
  /*! ZSTD_decompressFrame() :
606
873
  * @dctx must be properly initialized
@@ -610,8 +877,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
610
877
  void* dst, size_t dstCapacity,
611
878
  const void** srcPtr, size_t *srcSizePtr)
612
879
  {
613
- const BYTE* ip = (const BYTE*)(*srcPtr);
614
- BYTE* const ostart = (BYTE* const)dst;
880
+ const BYTE* const istart = (const BYTE*)(*srcPtr);
881
+ const BYTE* ip = istart;
882
+ BYTE* const ostart = (BYTE*)dst;
615
883
  BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
616
884
  BYTE* op = ostart;
617
885
  size_t remainingSrcSize = *srcSizePtr;
@@ -647,13 +915,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
647
915
  switch(blockProperties.blockType)
648
916
  {
649
917
  case bt_compressed:
650
- decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize, /* frame */ 1);
918
+ decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
651
919
  break;
652
920
  case bt_raw :
653
- decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
921
+ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
654
922
  break;
655
923
  case bt_rle :
656
- decodedSize = ZSTD_setRleBlock(op, oend-op, *ip, blockProperties.origSize);
924
+ decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
657
925
  break;
658
926
  case bt_reserved :
659
927
  default:
@@ -661,7 +929,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
661
929
  }
662
930
 
663
931
  if (ZSTD_isError(decodedSize)) return decodedSize;
664
- if (dctx->fParams.checksumFlag)
932
+ if (dctx->validateChecksum)
665
933
  XXH64_update(&dctx->xxhState, op, decodedSize);
666
934
  if (decodedSize != 0)
667
935
  op += decodedSize;
@@ -676,19 +944,21 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
676
944
  corruption_detected, "");
677
945
  }
678
946
  if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
679
- U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
680
- U32 checkRead;
681
947
  RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
682
- checkRead = MEM_readLE32(ip);
683
- RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
948
+ if (!dctx->forceIgnoreChecksum) {
949
+ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
950
+ U32 checkRead;
951
+ checkRead = MEM_readLE32(ip);
952
+ RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
953
+ }
684
954
  ip += 4;
685
955
  remainingSrcSize -= 4;
686
956
  }
687
-
957
+ ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
688
958
  /* Allow caller to get size read */
689
959
  *srcPtr = ip;
690
960
  *srcSizePtr = remainingSrcSize;
691
- return op-ostart;
961
+ return (size_t)(op-ostart);
692
962
  }
693
963
 
694
964
  static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
@@ -721,7 +991,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
721
991
  decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
722
992
  if (ZSTD_isError(decodedSize)) return decodedSize;
723
993
 
724
- assert(decodedSize <=- dstCapacity);
994
+ assert(decodedSize <= dstCapacity);
725
995
  dst = (BYTE*)dst + decodedSize;
726
996
  dstCapacity -= decodedSize;
727
997
 
@@ -753,7 +1023,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
753
1023
  * use this in all cases but ddict */
754
1024
  FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
755
1025
  }
756
- ZSTD_checkContinuity(dctx, dst);
1026
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
757
1027
 
758
1028
  { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
759
1029
  &src, &srcSize);
@@ -761,15 +1031,13 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
761
1031
  (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
762
1032
  && (moreThan1Frame==1),
763
1033
  srcSize_wrong,
764
- "at least one frame successfully completed, but following "
765
- "bytes are garbage: it's more likely to be a srcSize error, "
766
- "specifying more bytes than compressed size of frame(s). This "
767
- "error message replaces ERROR(prefix_unknown), which would be "
768
- "confusing, as the first header is actually correct. Note that "
769
- "one could be unlucky, it might be a corruption error instead, "
770
- "happening right at the place where we expect zstd magic "
771
- "bytes. But this is _much_ less likely than a srcSize field "
772
- "error.");
1034
+ "At least one frame successfully completed, "
1035
+ "but following bytes are garbage: "
1036
+ "it's more likely to be a srcSize error, "
1037
+ "specifying more input bytes than size of frame(s). "
1038
+ "Note: one could be unlucky, it might be a corruption error instead, "
1039
+ "happening right at the place where we expect zstd magic bytes. "
1040
+ "But this is _much_ less likely than a srcSize field error.");
773
1041
  if (ZSTD_isError(res)) return res;
774
1042
  assert(res <= dstCapacity);
775
1043
  if (res != 0)
@@ -781,7 +1049,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
781
1049
 
782
1050
  RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
783
1051
 
784
- return (BYTE*)dst - (BYTE*)dststart;
1052
+ return (size_t)((BYTE*)dst - (BYTE*)dststart);
785
1053
  }
786
1054
 
787
1055
  size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
@@ -798,7 +1066,7 @@ static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
798
1066
  switch (dctx->dictUses) {
799
1067
  default:
800
1068
  assert(0 /* Impossible */);
801
- /* fall-through */
1069
+ ZSTD_FALLTHROUGH;
802
1070
  case ZSTD_dont_use:
803
1071
  ZSTD_clearDict(dctx);
804
1072
  return NULL;
@@ -820,7 +1088,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
820
1088
  {
821
1089
  #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
822
1090
  size_t regenSize;
823
- ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1091
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem);
824
1092
  RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
825
1093
  regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
826
1094
  ZSTD_freeDCtx(dctx);
@@ -854,7 +1122,7 @@ static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t
854
1122
  return dctx->expected;
855
1123
  if (dctx->bType != bt_raw)
856
1124
  return dctx->expected;
857
- return MIN(MAX(inputSize, 1), dctx->expected);
1125
+ return BOUNDED(1, inputSize, dctx->expected);
858
1126
  }
859
1127
 
860
1128
  ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
@@ -862,7 +1130,9 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
862
1130
  {
863
1131
  default: /* should not happen */
864
1132
  assert(0);
1133
+ ZSTD_FALLTHROUGH;
865
1134
  case ZSTDds_getFrameHeaderSize:
1135
+ ZSTD_FALLTHROUGH;
866
1136
  case ZSTDds_decodeFrameHeader:
867
1137
  return ZSTDnit_frameHeader;
868
1138
  case ZSTDds_decodeBlockHeader:
@@ -874,6 +1144,7 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
874
1144
  case ZSTDds_checkChecksum:
875
1145
  return ZSTDnit_checksum;
876
1146
  case ZSTDds_decodeSkippableHeader:
1147
+ ZSTD_FALLTHROUGH;
877
1148
  case ZSTDds_skipFrame:
878
1149
  return ZSTDnit_skippableFrame;
879
1150
  }
@@ -890,7 +1161,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
890
1161
  DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
891
1162
  /* Sanity check */
892
1163
  RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
893
- if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
1164
+ ZSTD_checkContinuity(dctx, dst, dstCapacity);
1165
+
1166
+ dctx->processedCSize += srcSize;
894
1167
 
895
1168
  switch (dctx->stage)
896
1169
  {
@@ -899,21 +1172,21 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
899
1172
  if (dctx->format == ZSTD_f_zstd1) { /* allows header */
900
1173
  assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
901
1174
  if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
902
- memcpy(dctx->headerBuffer, src, srcSize);
1175
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
903
1176
  dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
904
1177
  dctx->stage = ZSTDds_decodeSkippableHeader;
905
1178
  return 0;
906
1179
  } }
907
1180
  dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
908
1181
  if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
909
- memcpy(dctx->headerBuffer, src, srcSize);
1182
+ ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
910
1183
  dctx->expected = dctx->headerSize - srcSize;
911
1184
  dctx->stage = ZSTDds_decodeFrameHeader;
912
1185
  return 0;
913
1186
 
914
1187
  case ZSTDds_decodeFrameHeader:
915
1188
  assert(src != NULL);
916
- memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
1189
+ ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
917
1190
  FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
918
1191
  dctx->expected = ZSTD_blockHeaderSize;
919
1192
  dctx->stage = ZSTDds_decodeBlockHeader;
@@ -955,7 +1228,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
955
1228
  {
956
1229
  case bt_compressed:
957
1230
  DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
958
- rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1);
1231
+ rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
959
1232
  dctx->expected = 0; /* Streaming not supported */
960
1233
  break;
961
1234
  case bt_raw :
@@ -977,7 +1250,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
977
1250
  RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
978
1251
  DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
979
1252
  dctx->decodedSize += rSize;
980
- if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
1253
+ if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
981
1254
  dctx->previousDstEnd = (char*)dst + rSize;
982
1255
 
983
1256
  /* Stay on the same stage until we are finished streaming the block. */
@@ -995,6 +1268,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
995
1268
  dctx->expected = 4;
996
1269
  dctx->stage = ZSTDds_checkChecksum;
997
1270
  } else {
1271
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
998
1272
  dctx->expected = 0; /* ends here */
999
1273
  dctx->stage = ZSTDds_getFrameHeaderSize;
1000
1274
  }
@@ -1007,10 +1281,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
1007
1281
 
1008
1282
  case ZSTDds_checkChecksum:
1009
1283
  assert(srcSize == 4); /* guaranteed by dctx->expected */
1010
- { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
1011
- U32 const check32 = MEM_readLE32(src);
1012
- DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
1013
- RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
1284
+ {
1285
+ if (dctx->validateChecksum) {
1286
+ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
1287
+ U32 const check32 = MEM_readLE32(src);
1288
+ DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
1289
+ RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
1290
+ }
1291
+ ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
1014
1292
  dctx->expected = 0;
1015
1293
  dctx->stage = ZSTDds_getFrameHeaderSize;
1016
1294
  return 0;
@@ -1019,7 +1297,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
1019
1297
  case ZSTDds_decodeSkippableHeader:
1020
1298
  assert(src != NULL);
1021
1299
  assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
1022
- memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
1300
+ ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
1023
1301
  dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
1024
1302
  dctx->stage = ZSTDds_skipFrame;
1025
1303
  return 0;
@@ -1075,7 +1353,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1075
1353
  workspace, workspaceSize);
1076
1354
  #else
1077
1355
  size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
1078
- dictPtr, dictEnd - dictPtr,
1356
+ dictPtr, (size_t)(dictEnd - dictPtr),
1079
1357
  workspace, workspaceSize);
1080
1358
  #endif
1081
1359
  RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
@@ -1084,40 +1362,46 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1084
1362
 
1085
1363
  { short offcodeNCount[MaxOff+1];
1086
1364
  unsigned offcodeMaxValue = MaxOff, offcodeLog;
1087
- size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
1365
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
1088
1366
  RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
1089
1367
  RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
1090
1368
  RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
1091
1369
  ZSTD_buildFSETable( entropy->OFTable,
1092
1370
  offcodeNCount, offcodeMaxValue,
1093
1371
  OF_base, OF_bits,
1094
- offcodeLog);
1372
+ offcodeLog,
1373
+ entropy->workspace, sizeof(entropy->workspace),
1374
+ /* bmi2 */0);
1095
1375
  dictPtr += offcodeHeaderSize;
1096
1376
  }
1097
1377
 
1098
1378
  { short matchlengthNCount[MaxML+1];
1099
1379
  unsigned matchlengthMaxValue = MaxML, matchlengthLog;
1100
- size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
1380
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
1101
1381
  RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
1102
1382
  RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
1103
1383
  RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
1104
1384
  ZSTD_buildFSETable( entropy->MLTable,
1105
1385
  matchlengthNCount, matchlengthMaxValue,
1106
1386
  ML_base, ML_bits,
1107
- matchlengthLog);
1387
+ matchlengthLog,
1388
+ entropy->workspace, sizeof(entropy->workspace),
1389
+ /* bmi2 */ 0);
1108
1390
  dictPtr += matchlengthHeaderSize;
1109
1391
  }
1110
1392
 
1111
1393
  { short litlengthNCount[MaxLL+1];
1112
1394
  unsigned litlengthMaxValue = MaxLL, litlengthLog;
1113
- size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
1395
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
1114
1396
  RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
1115
1397
  RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
1116
1398
  RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
1117
1399
  ZSTD_buildFSETable( entropy->LLTable,
1118
1400
  litlengthNCount, litlengthMaxValue,
1119
1401
  LL_base, LL_bits,
1120
- litlengthLog);
1402
+ litlengthLog,
1403
+ entropy->workspace, sizeof(entropy->workspace),
1404
+ /* bmi2 */ 0);
1121
1405
  dictPtr += litlengthHeaderSize;
1122
1406
  }
1123
1407
 
@@ -1131,7 +1415,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
1131
1415
  entropy->rep[i] = rep;
1132
1416
  } }
1133
1417
 
1134
- return dictPtr - (const BYTE*)dict;
1418
+ return (size_t)(dictPtr - (const BYTE*)dict);
1135
1419
  }
1136
1420
 
1137
1421
  static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
@@ -1158,8 +1442,12 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
1158
1442
  size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
1159
1443
  {
1160
1444
  assert(dctx != NULL);
1445
+ #if ZSTD_TRACE
1446
+ dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
1447
+ #endif
1161
1448
  dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
1162
1449
  dctx->stage = ZSTDds_getFrameHeaderSize;
1450
+ dctx->processedCSize = 0;
1163
1451
  dctx->decodedSize = 0;
1164
1452
  dctx->previousDstEnd = NULL;
1165
1453
  dctx->prefixStart = NULL;
@@ -1170,7 +1458,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
1170
1458
  dctx->dictID = 0;
1171
1459
  dctx->bType = bt_reserved;
1172
1460
  ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
1173
- memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
1461
+ ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
1174
1462
  dctx->LLTptr = dctx->entropy.LLTable;
1175
1463
  dctx->MLTptr = dctx->entropy.MLTable;
1176
1464
  dctx->OFTptr = dctx->entropy.OFTable;
@@ -1265,7 +1553,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
1265
1553
  ZSTD_DStream* ZSTD_createDStream(void)
1266
1554
  {
1267
1555
  DEBUGLOG(3, "ZSTD_createDStream");
1268
- return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
1556
+ return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
1269
1557
  }
1270
1558
 
1271
1559
  ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
@@ -1275,7 +1563,7 @@ ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
1275
1563
 
1276
1564
  ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
1277
1565
  {
1278
- return ZSTD_createDCtx_advanced(customMem);
1566
+ return ZSTD_createDCtx_internal(customMem);
1279
1567
  }
1280
1568
 
1281
1569
  size_t ZSTD_freeDStream(ZSTD_DStream* zds)
@@ -1373,6 +1661,16 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
1373
1661
  if (ddict) {
1374
1662
  dctx->ddict = ddict;
1375
1663
  dctx->dictUses = ZSTD_use_indefinitely;
1664
+ if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
1665
+ if (dctx->ddictSet == NULL) {
1666
+ dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
1667
+ if (!dctx->ddictSet) {
1668
+ RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
1669
+ }
1670
+ }
1671
+ assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */
1672
+ FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
1673
+ }
1376
1674
  }
1377
1675
  return 0;
1378
1676
  }
@@ -1394,7 +1692,7 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
1394
1692
 
1395
1693
  size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
1396
1694
  {
1397
- return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format);
1695
+ return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
1398
1696
  }
1399
1697
 
1400
1698
  ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
@@ -1411,8 +1709,16 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
1411
1709
  ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
1412
1710
  return bounds;
1413
1711
  case ZSTD_d_stableOutBuffer:
1414
- bounds.lowerBound = (int)ZSTD_obm_buffered;
1415
- bounds.upperBound = (int)ZSTD_obm_stable;
1712
+ bounds.lowerBound = (int)ZSTD_bm_buffered;
1713
+ bounds.upperBound = (int)ZSTD_bm_stable;
1714
+ return bounds;
1715
+ case ZSTD_d_forceIgnoreChecksum:
1716
+ bounds.lowerBound = (int)ZSTD_d_validateChecksum;
1717
+ bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
1718
+ return bounds;
1719
+ case ZSTD_d_refMultipleDDicts:
1720
+ bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
1721
+ bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
1416
1722
  return bounds;
1417
1723
  default:;
1418
1724
  }
@@ -1436,6 +1742,29 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
1436
1742
  RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
1437
1743
  }
1438
1744
 
1745
+ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
1746
+ {
1747
+ switch (param) {
1748
+ case ZSTD_d_windowLogMax:
1749
+ *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
1750
+ return 0;
1751
+ case ZSTD_d_format:
1752
+ *value = (int)dctx->format;
1753
+ return 0;
1754
+ case ZSTD_d_stableOutBuffer:
1755
+ *value = (int)dctx->outBufferMode;
1756
+ return 0;
1757
+ case ZSTD_d_forceIgnoreChecksum:
1758
+ *value = (int)dctx->forceIgnoreChecksum;
1759
+ return 0;
1760
+ case ZSTD_d_refMultipleDDicts:
1761
+ *value = (int)dctx->refMultipleDDicts;
1762
+ return 0;
1763
+ default:;
1764
+ }
1765
+ RETURN_ERROR(parameter_unsupported, "");
1766
+ }
1767
+
1439
1768
  size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
1440
1769
  {
1441
1770
  RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
@@ -1451,7 +1780,18 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
1451
1780
  return 0;
1452
1781
  case ZSTD_d_stableOutBuffer:
1453
1782
  CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
1454
- dctx->outBufferMode = (ZSTD_outBufferMode_e)value;
1783
+ dctx->outBufferMode = (ZSTD_bufferMode_e)value;
1784
+ return 0;
1785
+ case ZSTD_d_forceIgnoreChecksum:
1786
+ CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
1787
+ dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
1788
+ return 0;
1789
+ case ZSTD_d_refMultipleDDicts:
1790
+ CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
1791
+ if (dctx->staticSize != 0) {
1792
+ RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
1793
+ }
1794
+ dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
1455
1795
  return 0;
1456
1796
  default:;
1457
1797
  }
@@ -1469,8 +1809,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
1469
1809
  || (reset == ZSTD_reset_session_and_parameters) ) {
1470
1810
  RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
1471
1811
  ZSTD_clearDict(dctx);
1472
- dctx->format = ZSTD_f_zstd1;
1473
- dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
1812
+ ZSTD_DCtx_resetParameters(dctx);
1474
1813
  }
1475
1814
  return 0;
1476
1815
  }
@@ -1484,7 +1823,8 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
1484
1823
  size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
1485
1824
  {
1486
1825
  size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
1487
- unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
1826
+ /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
1827
+ unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
1488
1828
  unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
1489
1829
  size_t const minRBSize = (size_t) neededSize;
1490
1830
  RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
@@ -1524,7 +1864,7 @@ static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const ne
1524
1864
  {
1525
1865
  if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
1526
1866
  zds->oversizedDuration++;
1527
- else
1867
+ else
1528
1868
  zds->oversizedDuration = 0;
1529
1869
  }
1530
1870
 
@@ -1538,7 +1878,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const*
1538
1878
  {
1539
1879
  ZSTD_outBuffer const expect = zds->expectedOutBuffer;
1540
1880
  /* No requirement when ZSTD_obm_stable is not enabled. */
1541
- if (zds->outBufferMode != ZSTD_obm_stable)
1881
+ if (zds->outBufferMode != ZSTD_bm_stable)
1542
1882
  return 0;
1543
1883
  /* Any buffer is allowed in zdss_init, this must be the same for every other call until
1544
1884
  * the context is reset.
@@ -1548,7 +1888,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const*
1548
1888
  /* The buffer must match our expectation exactly. */
1549
1889
  if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
1550
1890
  return 0;
1551
- RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!");
1891
+ RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
1552
1892
  }
1553
1893
 
1554
1894
  /* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
@@ -1560,7 +1900,7 @@ static size_t ZSTD_decompressContinueStream(
1560
1900
  ZSTD_DStream* zds, char** op, char* oend,
1561
1901
  void const* src, size_t srcSize) {
1562
1902
  int const isSkipFrame = ZSTD_isSkipFrame(zds);
1563
- if (zds->outBufferMode == ZSTD_obm_buffered) {
1903
+ if (zds->outBufferMode == ZSTD_bm_buffered) {
1564
1904
  size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
1565
1905
  size_t const decodedSize = ZSTD_decompressContinue(zds,
1566
1906
  zds->outBuff + zds->outStart, dstSize, src, srcSize);
@@ -1573,14 +1913,14 @@ static size_t ZSTD_decompressContinueStream(
1573
1913
  }
1574
1914
  } else {
1575
1915
  /* Write directly into the output buffer */
1576
- size_t const dstSize = isSkipFrame ? 0 : oend - *op;
1916
+ size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
1577
1917
  size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
1578
1918
  FORWARD_IF_ERROR(decodedSize, "");
1579
1919
  *op += decodedSize;
1580
1920
  /* Flushing is not needed. */
1581
1921
  zds->streamStage = zdss_read;
1582
1922
  assert(*op <= oend);
1583
- assert(zds->outBufferMode == ZSTD_obm_stable);
1923
+ assert(zds->outBufferMode == ZSTD_bm_stable);
1584
1924
  }
1585
1925
  return 0;
1586
1926
  }
@@ -1618,10 +1958,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1618
1958
  DEBUGLOG(5, "stage zdss_init => transparent reset ");
1619
1959
  zds->streamStage = zdss_loadHeader;
1620
1960
  zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
1961
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
1621
1962
  zds->legacyVersion = 0;
1963
+ #endif
1622
1964
  zds->hostageByte = 0;
1623
1965
  zds->expectedOutBuffer = *output;
1624
- /* fall-through */
1966
+ ZSTD_FALLTHROUGH;
1625
1967
 
1626
1968
  case zdss_loadHeader :
1627
1969
  DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
@@ -1635,6 +1977,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1635
1977
  } }
1636
1978
  #endif
1637
1979
  { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
1980
+ if (zds->refMultipleDDicts && zds->ddictSet) {
1981
+ ZSTD_DCtx_selectFrameDDict(zds);
1982
+ }
1638
1983
  DEBUGLOG(5, "header size : %u", (U32)hSize);
1639
1984
  if (ZSTD_isError(hSize)) {
1640
1985
  #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
@@ -1663,14 +2008,14 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1663
2008
  assert(iend >= ip);
1664
2009
  if (toLoad > remainingInput) { /* not enough input to load full header */
1665
2010
  if (remainingInput > 0) {
1666
- memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
2011
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
1667
2012
  zds->lhSize += remainingInput;
1668
2013
  }
1669
2014
  input->pos = input->size;
1670
2015
  return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
1671
2016
  }
1672
2017
  assert(ip != NULL);
1673
- memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
2018
+ ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
1674
2019
  break;
1675
2020
  } }
1676
2021
 
@@ -1678,10 +2023,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1678
2023
  if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
1679
2024
  && zds->fParams.frameType != ZSTD_skippableFrame
1680
2025
  && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
1681
- size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
2026
+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
1682
2027
  if (cSize <= (size_t)(iend-istart)) {
1683
2028
  /* shortcut : using single-pass mode */
1684
- size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds));
2029
+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
1685
2030
  if (ZSTD_isError(decompressedSize)) return decompressedSize;
1686
2031
  DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
1687
2032
  ip = istart + cSize;
@@ -1693,7 +2038,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1693
2038
  } }
1694
2039
 
1695
2040
  /* Check output buffer is large enough for ZSTD_odm_stable. */
1696
- if (zds->outBufferMode == ZSTD_obm_stable
2041
+ if (zds->outBufferMode == ZSTD_bm_stable
1697
2042
  && zds->fParams.frameType != ZSTD_skippableFrame
1698
2043
  && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
1699
2044
  && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
@@ -1723,7 +2068,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1723
2068
 
1724
2069
  /* Adapt buffer sizes to frame header instructions */
1725
2070
  { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
1726
- size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered
2071
+ size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
1727
2072
  ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
1728
2073
  : 0;
1729
2074
 
@@ -1731,7 +2076,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1731
2076
 
1732
2077
  { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize);
1733
2078
  int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
1734
-
2079
+
1735
2080
  if (tooSmall || tooLarge) {
1736
2081
  size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
1737
2082
  DEBUGLOG(4, "inBuff : from %u to %u",
@@ -1745,10 +2090,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1745
2090
  bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
1746
2091
  memory_allocation, "");
1747
2092
  } else {
1748
- ZSTD_free(zds->inBuff, zds->customMem);
2093
+ ZSTD_customFree(zds->inBuff, zds->customMem);
1749
2094
  zds->inBuffSize = 0;
1750
2095
  zds->outBuffSize = 0;
1751
- zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
2096
+ zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
1752
2097
  RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
1753
2098
  }
1754
2099
  zds->inBuffSize = neededInBuffSize;
@@ -1756,11 +2101,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1756
2101
  zds->outBuffSize = neededOutBuffSize;
1757
2102
  } } }
1758
2103
  zds->streamStage = zdss_read;
1759
- /* fall-through */
2104
+ ZSTD_FALLTHROUGH;
1760
2105
 
1761
2106
  case zdss_read:
1762
2107
  DEBUGLOG(5, "stage zdss_read");
1763
- { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip);
2108
+ { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
1764
2109
  DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
1765
2110
  if (neededInSize==0) { /* end of frame */
1766
2111
  zds->streamStage = zdss_init;
@@ -1775,7 +2120,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1775
2120
  } }
1776
2121
  if (ip==iend) { someMoreWork = 0; break; } /* no more input */
1777
2122
  zds->streamStage = zdss_load;
1778
- /* fall-through */
2123
+ ZSTD_FALLTHROUGH;
1779
2124
 
1780
2125
  case zdss_load:
1781
2126
  { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
@@ -1790,7 +2135,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1790
2135
  RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
1791
2136
  corruption_detected,
1792
2137
  "should never happen");
1793
- loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
2138
+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
1794
2139
  }
1795
2140
  ip += loadedSize;
1796
2141
  zds->inPos += loadedSize;
@@ -1804,7 +2149,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
1804
2149
  }
1805
2150
  case zdss_flush:
1806
2151
  { size_t const toFlushSize = zds->outEnd - zds->outStart;
1807
- size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
2152
+ size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
1808
2153
  op += flushedSize;
1809
2154
  zds->outStart += flushedSize;
1810
2155
  if (flushedSize == toFlushSize) { /* flush completed */