zstd-ruby 1.5.1.1 → 1.5.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (117) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/README.md +78 -5
  4. data/Rakefile +8 -2
  5. data/ext/zstdruby/common.h +15 -0
  6. data/ext/zstdruby/extconf.rb +1 -1
  7. data/ext/zstdruby/libzstd/common/allocations.h +55 -0
  8. data/ext/zstdruby/libzstd/common/bits.h +200 -0
  9. data/ext/zstdruby/libzstd/common/bitstream.h +19 -60
  10. data/ext/zstdruby/libzstd/common/compiler.h +26 -3
  11. data/ext/zstdruby/libzstd/common/cpu.h +1 -1
  12. data/ext/zstdruby/libzstd/common/debug.c +1 -1
  13. data/ext/zstdruby/libzstd/common/debug.h +1 -1
  14. data/ext/zstdruby/libzstd/common/entropy_common.c +12 -40
  15. data/ext/zstdruby/libzstd/common/error_private.c +9 -2
  16. data/ext/zstdruby/libzstd/common/error_private.h +1 -1
  17. data/ext/zstdruby/libzstd/common/fse.h +5 -83
  18. data/ext/zstdruby/libzstd/common/fse_decompress.c +7 -99
  19. data/ext/zstdruby/libzstd/common/huf.h +65 -156
  20. data/ext/zstdruby/libzstd/common/mem.h +39 -46
  21. data/ext/zstdruby/libzstd/common/pool.c +37 -16
  22. data/ext/zstdruby/libzstd/common/pool.h +9 -3
  23. data/ext/zstdruby/libzstd/common/portability_macros.h +28 -3
  24. data/ext/zstdruby/libzstd/common/threading.c +68 -14
  25. data/ext/zstdruby/libzstd/common/threading.h +5 -10
  26. data/ext/zstdruby/libzstd/common/xxhash.c +2 -2
  27. data/ext/zstdruby/libzstd/common/xxhash.h +8 -8
  28. data/ext/zstdruby/libzstd/common/zstd_common.c +1 -36
  29. data/ext/zstdruby/libzstd/common/zstd_deps.h +1 -1
  30. data/ext/zstdruby/libzstd/common/zstd_internal.h +20 -122
  31. data/ext/zstdruby/libzstd/common/zstd_trace.h +3 -3
  32. data/ext/zstdruby/libzstd/compress/clevels.h +1 -1
  33. data/ext/zstdruby/libzstd/compress/fse_compress.c +7 -124
  34. data/ext/zstdruby/libzstd/compress/hist.c +1 -1
  35. data/ext/zstdruby/libzstd/compress/hist.h +1 -1
  36. data/ext/zstdruby/libzstd/compress/huf_compress.c +234 -169
  37. data/ext/zstdruby/libzstd/compress/zstd_compress.c +1317 -594
  38. data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +272 -165
  39. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.c +115 -39
  40. data/ext/zstdruby/libzstd/compress/zstd_compress_literals.h +16 -8
  41. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +13 -13
  42. data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.h +1 -1
  43. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +25 -21
  44. data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.h +1 -1
  45. data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +162 -82
  46. data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +95 -33
  47. data/ext/zstdruby/libzstd/compress/zstd_double_fast.h +3 -2
  48. data/ext/zstdruby/libzstd/compress/zstd_fast.c +434 -149
  49. data/ext/zstdruby/libzstd/compress/zstd_fast.h +3 -2
  50. data/ext/zstdruby/libzstd/compress/zstd_lazy.c +405 -348
  51. data/ext/zstdruby/libzstd/compress/zstd_lazy.h +4 -2
  52. data/ext/zstdruby/libzstd/compress/zstd_ldm.c +9 -7
  53. data/ext/zstdruby/libzstd/compress/zstd_ldm.h +1 -1
  54. data/ext/zstdruby/libzstd/compress/zstd_ldm_geartab.h +1 -1
  55. data/ext/zstdruby/libzstd/compress/zstd_opt.c +149 -100
  56. data/ext/zstdruby/libzstd/compress/zstd_opt.h +1 -1
  57. data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +32 -16
  58. data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +5 -2
  59. data/ext/zstdruby/libzstd/decompress/huf_decompress.c +434 -441
  60. data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +42 -37
  61. data/ext/zstdruby/libzstd/decompress/zstd_ddict.c +4 -4
  62. data/ext/zstdruby/libzstd/decompress/zstd_ddict.h +1 -1
  63. data/ext/zstdruby/libzstd/decompress/zstd_decompress.c +205 -80
  64. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.c +201 -81
  65. data/ext/zstdruby/libzstd/decompress/zstd_decompress_block.h +6 -1
  66. data/ext/zstdruby/libzstd/decompress/zstd_decompress_internal.h +4 -2
  67. data/ext/zstdruby/libzstd/dictBuilder/cover.c +19 -15
  68. data/ext/zstdruby/libzstd/dictBuilder/cover.h +1 -1
  69. data/ext/zstdruby/libzstd/dictBuilder/fastcover.c +2 -2
  70. data/ext/zstdruby/libzstd/dictBuilder/zdict.c +13 -91
  71. data/ext/zstdruby/libzstd/zdict.h +53 -31
  72. data/ext/zstdruby/libzstd/zstd.h +580 -135
  73. data/ext/zstdruby/libzstd/zstd_errors.h +27 -8
  74. data/ext/zstdruby/main.c +20 -0
  75. data/ext/zstdruby/skippable_frame.c +63 -0
  76. data/ext/zstdruby/streaming_compress.c +177 -0
  77. data/ext/zstdruby/streaming_compress.h +5 -0
  78. data/ext/zstdruby/streaming_decompress.c +123 -0
  79. data/ext/zstdruby/zstdruby.c +113 -31
  80. data/lib/zstd-ruby/version.rb +1 -1
  81. data/lib/zstd-ruby.rb +0 -1
  82. data/zstd-ruby.gemspec +1 -1
  83. metadata +11 -37
  84. data/.github/dependabot.yml +0 -8
  85. data/.github/workflows/ruby.yml +0 -35
  86. data/ext/zstdruby/libzstd/.gitignore +0 -3
  87. data/ext/zstdruby/libzstd/BUCK +0 -232
  88. data/ext/zstdruby/libzstd/Makefile +0 -357
  89. data/ext/zstdruby/libzstd/README.md +0 -217
  90. data/ext/zstdruby/libzstd/deprecated/zbuff.h +0 -214
  91. data/ext/zstdruby/libzstd/deprecated/zbuff_common.c +0 -26
  92. data/ext/zstdruby/libzstd/deprecated/zbuff_compress.c +0 -167
  93. data/ext/zstdruby/libzstd/deprecated/zbuff_decompress.c +0 -75
  94. data/ext/zstdruby/libzstd/dll/example/Makefile +0 -48
  95. data/ext/zstdruby/libzstd/dll/example/README.md +0 -63
  96. data/ext/zstdruby/libzstd/dll/example/build_package.bat +0 -20
  97. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.sln +0 -25
  98. data/ext/zstdruby/libzstd/dll/example/fullbench-dll.vcxproj +0 -181
  99. data/ext/zstdruby/libzstd/legacy/zstd_legacy.h +0 -415
  100. data/ext/zstdruby/libzstd/legacy/zstd_v01.c +0 -2158
  101. data/ext/zstdruby/libzstd/legacy/zstd_v01.h +0 -94
  102. data/ext/zstdruby/libzstd/legacy/zstd_v02.c +0 -3518
  103. data/ext/zstdruby/libzstd/legacy/zstd_v02.h +0 -93
  104. data/ext/zstdruby/libzstd/legacy/zstd_v03.c +0 -3160
  105. data/ext/zstdruby/libzstd/legacy/zstd_v03.h +0 -93
  106. data/ext/zstdruby/libzstd/legacy/zstd_v04.c +0 -3647
  107. data/ext/zstdruby/libzstd/legacy/zstd_v04.h +0 -142
  108. data/ext/zstdruby/libzstd/legacy/zstd_v05.c +0 -4050
  109. data/ext/zstdruby/libzstd/legacy/zstd_v05.h +0 -162
  110. data/ext/zstdruby/libzstd/legacy/zstd_v06.c +0 -4154
  111. data/ext/zstdruby/libzstd/legacy/zstd_v06.h +0 -172
  112. data/ext/zstdruby/libzstd/legacy/zstd_v07.c +0 -4541
  113. data/ext/zstdruby/libzstd/legacy/zstd_v07.h +0 -187
  114. data/ext/zstdruby/libzstd/libzstd.mk +0 -185
  115. data/ext/zstdruby/libzstd/libzstd.pc.in +0 -16
  116. data/ext/zstdruby/libzstd/modulemap/module.modulemap +0 -4
  117. data/ext/zstdruby/zstdruby.h +0 -6
@@ -1,6 +1,6 @@
1
1
  /* ******************************************************************
2
2
  * Common functions of New Generation Entropy library
3
- * Copyright (c) Yann Collet, Facebook, Inc.
3
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
4
4
  *
5
5
  * You can contact the author at :
6
6
  * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -19,8 +19,8 @@
19
19
  #include "error_private.h" /* ERR_*, ERROR */
20
20
  #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
21
21
  #include "fse.h"
22
- #define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
23
22
  #include "huf.h"
23
+ #include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
24
24
 
25
25
 
26
26
  /*=== Version ===*/
@@ -38,34 +38,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
38
38
  /*-**************************************************************
39
39
  * FSE NCount encoding-decoding
40
40
  ****************************************************************/
41
- static U32 FSE_ctz(U32 val)
42
- {
43
- assert(val != 0);
44
- {
45
- # if defined(_MSC_VER) /* Visual */
46
- if (val != 0) {
47
- unsigned long r;
48
- _BitScanForward(&r, val);
49
- return (unsigned)r;
50
- } else {
51
- /* Should not reach this code path */
52
- __assume(0);
53
- }
54
- # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
55
- return __builtin_ctz(val);
56
- # elif defined(__ICCARM__) /* IAR Intrinsic */
57
- return __CTZ(val);
58
- # else /* Software version */
59
- U32 count = 0;
60
- while ((val & 1) == 0) {
61
- val >>= 1;
62
- ++count;
63
- }
64
- return count;
65
- # endif
66
- }
67
- }
68
-
69
41
  FORCE_INLINE_TEMPLATE
70
42
  size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
71
43
  const void* headerBuffer, size_t hbSize)
@@ -113,7 +85,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
113
85
  * repeat.
114
86
  * Avoid UB by setting the high bit to 1.
115
87
  */
116
- int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
88
+ int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
117
89
  while (repeats >= 12) {
118
90
  charnum += 3 * 12;
119
91
  if (LIKELY(ip <= iend-7)) {
@@ -124,7 +96,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
124
96
  ip = iend - 4;
125
97
  }
126
98
  bitStream = MEM_readLE32(ip) >> bitCount;
127
- repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
99
+ repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
128
100
  }
129
101
  charnum += 3 * repeats;
130
102
  bitStream >>= 2 * repeats;
@@ -189,7 +161,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
189
161
  * know that threshold > 1.
190
162
  */
191
163
  if (remaining <= 1) break;
192
- nbBits = BIT_highbit32(remaining) + 1;
164
+ nbBits = ZSTD_highbit32(remaining) + 1;
193
165
  threshold = 1 << (nbBits - 1);
194
166
  }
195
167
  if (charnum >= maxSV1) break;
@@ -264,7 +236,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
264
236
  const void* src, size_t srcSize)
265
237
  {
266
238
  U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
267
- return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
239
+ return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
268
240
  }
269
241
 
270
242
  FORCE_INLINE_TEMPLATE size_t
@@ -312,14 +284,14 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
312
284
  if (weightTotal == 0) return ERROR(corruption_detected);
313
285
 
314
286
  /* get last non-null symbol weight (implied, total must be 2^n) */
315
- { U32 const tableLog = BIT_highbit32(weightTotal) + 1;
287
+ { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
316
288
  if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
317
289
  *tableLogPtr = tableLog;
318
290
  /* determine last weight */
319
291
  { U32 const total = 1 << tableLog;
320
292
  U32 const rest = total - weightTotal;
321
- U32 const verif = 1 << BIT_highbit32(rest);
322
- U32 const lastWeight = BIT_highbit32(rest) + 1;
293
+ U32 const verif = 1 << ZSTD_highbit32(rest);
294
+ U32 const lastWeight = ZSTD_highbit32(rest) + 1;
323
295
  if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
324
296
  huffWeight[oSize] = (BYTE)lastWeight;
325
297
  rankStats[lastWeight]++;
@@ -356,13 +328,13 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
356
328
  U32* nbSymbolsPtr, U32* tableLogPtr,
357
329
  const void* src, size_t srcSize,
358
330
  void* workSpace, size_t wkspSize,
359
- int bmi2)
331
+ int flags)
360
332
  {
361
333
  #if DYNAMIC_BMI2
362
- if (bmi2) {
334
+ if (flags & HUF_flags_bmi2) {
363
335
  return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
364
336
  }
365
337
  #endif
366
- (void)bmi2;
338
+ (void)flags;
367
339
  return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
368
340
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -27,9 +27,11 @@ const char* ERR_getErrorString(ERR_enum code)
27
27
  case PREFIX(version_unsupported): return "Version not supported";
28
28
  case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
29
29
  case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
30
- case PREFIX(corruption_detected): return "Corrupted block detected";
30
+ case PREFIX(corruption_detected): return "Data corruption detected";
31
31
  case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
32
+ case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
32
33
  case PREFIX(parameter_unsupported): return "Unsupported parameter";
34
+ case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
33
35
  case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
34
36
  case PREFIX(init_missing): return "Context should be init first";
35
37
  case PREFIX(memory_allocation): return "Allocation error : not enough memory";
@@ -38,17 +40,22 @@ const char* ERR_getErrorString(ERR_enum code)
38
40
  case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
39
41
  case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
40
42
  case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
43
+ case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
41
44
  case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
42
45
  case PREFIX(dictionary_wrong): return "Dictionary mismatch";
43
46
  case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
44
47
  case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
45
48
  case PREFIX(srcSize_wrong): return "Src size is incorrect";
46
49
  case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
50
+ case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
51
+ case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
47
52
  /* following error codes are not stable and may be removed or changed in a future version */
48
53
  case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
49
54
  case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
50
55
  case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
51
56
  case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
57
+ case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
58
+ case PREFIX(externalSequences_invalid): return "External sequences are not valid";
52
59
  case PREFIX(maxCode):
53
60
  default: return notErrorCode;
54
61
  }
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -1,7 +1,7 @@
1
1
  /* ******************************************************************
2
2
  * FSE : Finite State Entropy codec
3
3
  * Public Prototypes declaration
4
- * Copyright (c) Yann Collet, Facebook, Inc.
4
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
5
5
  *
6
6
  * You can contact the author at :
7
7
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -53,34 +53,6 @@ extern "C" {
53
53
  FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
54
54
 
55
55
 
56
- /*-****************************************
57
- * FSE simple functions
58
- ******************************************/
59
- /*! FSE_compress() :
60
- Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
61
- 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
62
- @return : size of compressed data (<= dstCapacity).
63
- Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
64
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
65
- if FSE_isError(return), compression failed (more details using FSE_getErrorName())
66
- */
67
- FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
68
- const void* src, size_t srcSize);
69
-
70
- /*! FSE_decompress():
71
- Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
72
- into already allocated destination buffer 'dst', of size 'dstCapacity'.
73
- @return : size of regenerated data (<= maxDstSize),
74
- or an error code, which can be tested using FSE_isError() .
75
-
76
- ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
77
- Why ? : making this distinction requires a header.
78
- Header management is intentionally delegated to the user layer, which can better manage special cases.
79
- */
80
- FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
81
- const void* cSrc, size_t cSrcSize);
82
-
83
-
84
56
  /*-*****************************************
85
57
  * Tool functions
86
58
  ******************************************/
@@ -91,20 +63,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return
91
63
  FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
92
64
 
93
65
 
94
- /*-*****************************************
95
- * FSE advanced functions
96
- ******************************************/
97
- /*! FSE_compress2() :
98
- Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
99
- Both parameters can be defined as '0' to mean : use default value
100
- @return : size of compressed data
101
- Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
102
- if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
103
- if FSE_isError(return), it's an error code.
104
- */
105
- FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
106
-
107
-
108
66
  /*-*****************************************
109
67
  * FSE detailed API
110
68
  ******************************************/
@@ -164,8 +122,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
164
122
  /*! Constructor and Destructor of FSE_CTable.
165
123
  Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
166
124
  typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
167
- FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
168
- FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
169
125
 
170
126
  /*! FSE_buildCTable():
171
127
  Builds `ct`, which must be already allocated, using FSE_createCTable().
@@ -241,23 +197,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
241
197
  unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
242
198
  const void* rBuffer, size_t rBuffSize, int bmi2);
243
199
 
244
- /*! Constructor and Destructor of FSE_DTable.
245
- Note that its size depends on 'tableLog' */
246
200
  typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
247
- FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
248
- FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
249
-
250
- /*! FSE_buildDTable():
251
- Builds 'dt', which must be already allocated, using FSE_createDTable().
252
- return : 0, or an errorCode, which can be tested using FSE_isError() */
253
- FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
254
-
255
- /*! FSE_decompress_usingDTable():
256
- Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
257
- into `dst` which must be already allocated.
258
- @return : size of regenerated data (necessarily <= `dstCapacity`),
259
- or an errorCode, which can be tested using FSE_isError() */
260
- FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
261
201
 
262
202
  /*!
263
203
  Tutorial :
@@ -320,16 +260,6 @@ If there is an error, the function will return an error code, which can be teste
320
260
  unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
321
261
  /**< same as FSE_optimalTableLog(), which used `minus==2` */
322
262
 
323
- /* FSE_compress_wksp() :
324
- * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
325
- * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
326
- */
327
- #define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
328
- size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
329
-
330
- size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
331
- /**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
332
-
333
263
  size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
334
264
  /**< build a fake FSE_CTable, designed to compress always the same symbolValue */
335
265
 
@@ -347,19 +277,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
347
277
  FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
348
278
  /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
349
279
 
350
- size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
351
- /**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
352
-
353
- size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
354
- /**< build a fake FSE_DTable, designed to always generate the same symbolValue */
355
-
356
- #define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
280
+ #define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
357
281
  #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
358
- size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
359
- /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
360
-
361
282
  size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
362
- /**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
283
+ /**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
284
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
363
285
 
364
286
  typedef enum {
365
287
  FSE_repeat_none, /**< Cannot use the previous table */
@@ -555,7 +477,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
555
477
 
556
478
  /* FSE_getMaxNbBits() :
557
479
  * Approximate maximum cost of a symbol, in bits.
558
- * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
480
+ * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
559
481
  * note 1 : assume symbolValue is valid (<= maxSymbolValue)
560
482
  * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
561
483
  MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
@@ -1,6 +1,6 @@
1
1
  /* ******************************************************************
2
2
  * FSE : Finite State Entropy decoder
3
- * Copyright (c) Yann Collet, Facebook, Inc.
3
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
4
4
  *
5
5
  * You can contact the author at :
6
6
  * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -24,6 +24,7 @@
24
24
  #include "error_private.h"
25
25
  #define ZSTD_DEPS_NEED_MALLOC
26
26
  #include "zstd_deps.h"
27
+ #include "bits.h" /* ZSTD_highbit32 */
27
28
 
28
29
 
29
30
  /* **************************************************************
@@ -55,19 +56,6 @@
55
56
  #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
56
57
  #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
57
58
 
58
-
59
- /* Function templates */
60
- FSE_DTable* FSE_createDTable (unsigned tableLog)
61
- {
62
- if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
63
- return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
64
- }
65
-
66
- void FSE_freeDTable (FSE_DTable* dt)
67
- {
68
- ZSTD_free(dt);
69
- }
70
-
71
59
  static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
72
60
  {
73
61
  void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
@@ -127,10 +115,10 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
127
115
  }
128
116
  }
129
117
  /* Now we spread those positions across the table.
130
- * The benefit of doing it in two stages is that we avoid the the
118
+ * The benefit of doing it in two stages is that we avoid the
131
119
  * variable size inner loop, which caused lots of branch misses.
132
120
  * Now we can run through all the positions without any branch misses.
133
- * We unroll the loop twice, since that is what emperically worked best.
121
+ * We unroll the loop twice, since that is what empirically worked best.
134
122
  */
135
123
  {
136
124
  size_t position = 0;
@@ -166,7 +154,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
166
154
  for (u=0; u<tableSize; u++) {
167
155
  FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
168
156
  U32 const nextState = symbolNext[symbol]++;
169
- tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
157
+ tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
170
158
  tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
171
159
  } }
172
160
 
@@ -184,49 +172,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi
184
172
  /*-*******************************************************
185
173
  * Decompression (Byte symbols)
186
174
  *********************************************************/
187
- size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
188
- {
189
- void* ptr = dt;
190
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
191
- void* dPtr = dt + 1;
192
- FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
193
-
194
- DTableH->tableLog = 0;
195
- DTableH->fastMode = 0;
196
-
197
- cell->newState = 0;
198
- cell->symbol = symbolValue;
199
- cell->nbBits = 0;
200
-
201
- return 0;
202
- }
203
-
204
-
205
- size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
206
- {
207
- void* ptr = dt;
208
- FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
209
- void* dPtr = dt + 1;
210
- FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
211
- const unsigned tableSize = 1 << nbBits;
212
- const unsigned tableMask = tableSize - 1;
213
- const unsigned maxSV1 = tableMask+1;
214
- unsigned s;
215
-
216
- /* Sanity checks */
217
- if (nbBits < 1) return ERROR(GENERIC); /* min size */
218
-
219
- /* Build Decoding Table */
220
- DTableH->tableLog = (U16)nbBits;
221
- DTableH->fastMode = 1;
222
- for (s=0; s<maxSV1; s++) {
223
- dinfo[s].newState = 0;
224
- dinfo[s].symbol = (BYTE)s;
225
- dinfo[s].nbBits = (BYTE)nbBits;
226
- }
227
-
228
- return 0;
229
- }
230
175
 
231
176
  FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
232
177
  void* dst, size_t maxDstSize,
@@ -290,26 +235,6 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
290
235
  return op-ostart;
291
236
  }
292
237
 
293
-
294
- size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
295
- const void* cSrc, size_t cSrcSize,
296
- const FSE_DTable* dt)
297
- {
298
- const void* ptr = dt;
299
- const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
300
- const U32 fastMode = DTableH->fastMode;
301
-
302
- /* select fast mode (static) */
303
- if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
304
- return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
305
- }
306
-
307
-
308
- size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
309
- {
310
- return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
311
- }
312
-
313
238
  typedef struct {
314
239
  short ncount[FSE_MAX_SYMBOL_VALUE + 1];
315
240
  FSE_DTable dtable[1]; /* Dynamically sized */
@@ -342,7 +267,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
342
267
  }
343
268
 
344
269
  if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
345
- workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
270
+ assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
271
+ workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
346
272
  wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
347
273
 
348
274
  CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
@@ -382,22 +308,4 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc,
382
308
  return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
383
309
  }
384
310
 
385
-
386
- typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
387
-
388
- #ifndef ZSTD_NO_UNUSED_FUNCTIONS
389
- size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
390
- U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
391
- return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
392
- }
393
-
394
- size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
395
- {
396
- /* Static analyzer seems unable to understand this table will be properly initialized later */
397
- U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
398
- return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
399
- }
400
- #endif
401
-
402
-
403
311
  #endif /* FSE_COMMONDEFS_ONLY */