extzstd 0.1.1 → 0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +18 -0
  3. data/README.md +15 -50
  4. data/contrib/zstd/CONTRIBUTING.md +1 -1
  5. data/contrib/zstd/COPYING +339 -0
  6. data/contrib/zstd/Makefile +82 -51
  7. data/contrib/zstd/NEWS +92 -5
  8. data/contrib/zstd/README.md +50 -41
  9. data/contrib/zstd/appveyor.yml +164 -102
  10. data/contrib/zstd/circle.yml +10 -22
  11. data/contrib/zstd/lib/BUCK +31 -10
  12. data/contrib/zstd/lib/Makefile +57 -31
  13. data/contrib/zstd/lib/README.md +68 -37
  14. data/contrib/zstd/lib/common/bitstream.h +130 -76
  15. data/contrib/zstd/lib/common/compiler.h +86 -0
  16. data/contrib/zstd/lib/common/error_private.c +15 -11
  17. data/contrib/zstd/lib/common/error_private.h +8 -8
  18. data/contrib/zstd/lib/common/fse.h +19 -9
  19. data/contrib/zstd/lib/common/fse_decompress.c +3 -22
  20. data/contrib/zstd/lib/common/huf.h +68 -26
  21. data/contrib/zstd/lib/common/mem.h +23 -35
  22. data/contrib/zstd/lib/common/pool.c +123 -63
  23. data/contrib/zstd/lib/common/pool.h +19 -10
  24. data/contrib/zstd/lib/common/threading.c +11 -16
  25. data/contrib/zstd/lib/common/threading.h +52 -33
  26. data/contrib/zstd/lib/common/xxhash.c +28 -22
  27. data/contrib/zstd/lib/common/zstd_common.c +40 -27
  28. data/contrib/zstd/lib/common/zstd_errors.h +43 -34
  29. data/contrib/zstd/lib/common/zstd_internal.h +131 -123
  30. data/contrib/zstd/lib/compress/fse_compress.c +17 -33
  31. data/contrib/zstd/lib/compress/huf_compress.c +15 -9
  32. data/contrib/zstd/lib/compress/zstd_compress.c +2096 -2363
  33. data/contrib/zstd/lib/compress/zstd_compress_internal.h +462 -0
  34. data/contrib/zstd/lib/compress/zstd_double_fast.c +309 -0
  35. data/contrib/zstd/lib/compress/zstd_double_fast.h +29 -0
  36. data/contrib/zstd/lib/compress/zstd_fast.c +243 -0
  37. data/contrib/zstd/lib/compress/zstd_fast.h +31 -0
  38. data/contrib/zstd/lib/compress/zstd_lazy.c +765 -0
  39. data/contrib/zstd/lib/compress/zstd_lazy.h +39 -0
  40. data/contrib/zstd/lib/compress/zstd_ldm.c +707 -0
  41. data/contrib/zstd/lib/compress/zstd_ldm.h +68 -0
  42. data/contrib/zstd/lib/compress/zstd_opt.c +785 -0
  43. data/contrib/zstd/lib/compress/zstd_opt.h +19 -908
  44. data/contrib/zstd/lib/compress/zstdmt_compress.c +737 -327
  45. data/contrib/zstd/lib/compress/zstdmt_compress.h +88 -26
  46. data/contrib/zstd/lib/decompress/huf_decompress.c +158 -50
  47. data/contrib/zstd/lib/decompress/zstd_decompress.c +884 -699
  48. data/contrib/zstd/lib/deprecated/zbuff.h +5 -4
  49. data/contrib/zstd/lib/deprecated/zbuff_common.c +5 -5
  50. data/contrib/zstd/lib/deprecated/zbuff_compress.c +6 -4
  51. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +5 -4
  52. data/contrib/zstd/lib/dictBuilder/cover.c +93 -77
  53. data/contrib/zstd/lib/dictBuilder/zdict.c +107 -92
  54. data/contrib/zstd/lib/dictBuilder/zdict.h +112 -102
  55. data/contrib/zstd/lib/legacy/zstd_legacy.h +9 -4
  56. data/contrib/zstd/lib/legacy/zstd_v01.c +7 -6
  57. data/contrib/zstd/lib/legacy/zstd_v01.h +5 -4
  58. data/contrib/zstd/lib/legacy/zstd_v02.c +27 -99
  59. data/contrib/zstd/lib/legacy/zstd_v02.h +5 -4
  60. data/contrib/zstd/lib/legacy/zstd_v03.c +26 -98
  61. data/contrib/zstd/lib/legacy/zstd_v03.h +5 -4
  62. data/contrib/zstd/lib/legacy/zstd_v04.c +22 -91
  63. data/contrib/zstd/lib/legacy/zstd_v04.h +5 -4
  64. data/contrib/zstd/lib/legacy/zstd_v05.c +23 -99
  65. data/contrib/zstd/lib/legacy/zstd_v05.h +5 -4
  66. data/contrib/zstd/lib/legacy/zstd_v06.c +22 -96
  67. data/contrib/zstd/lib/legacy/zstd_v06.h +5 -4
  68. data/contrib/zstd/lib/legacy/zstd_v07.c +19 -95
  69. data/contrib/zstd/lib/legacy/zstd_v07.h +5 -4
  70. data/contrib/zstd/lib/zstd.h +895 -271
  71. data/ext/extconf.rb +11 -2
  72. data/ext/extzstd.c +45 -128
  73. data/ext/extzstd.h +74 -31
  74. data/ext/extzstd_stream.c +401 -142
  75. data/ext/zstd_common.c +5 -0
  76. data/ext/zstd_compress.c +8 -0
  77. data/ext/zstd_decompress.c +1 -0
  78. data/ext/zstd_dictbuilder.c +2 -0
  79. data/lib/extzstd/version.rb +1 -1
  80. data/lib/extzstd.rb +48 -1
  81. data/test/test_basic.rb +9 -1
  82. metadata +17 -7
  83. data/HISTORY.ja +0 -10
  84. data/contrib/zstd/LICENSE-examples +0 -11
  85. data/contrib/zstd/PATENTS +0 -33
@@ -1,10 +1,11 @@
1
- /**
1
+ /*
2
2
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTDMT_COMPRESS_H
@@ -15,31 +16,41 @@
15
16
  #endif
16
17
 
17
18
 
18
- /* Note : All prototypes defined in this file shall be considered experimental.
19
- * There is no guarantee of API continuity (yet) on any of these prototypes */
19
+ /* Note : This is an internal API.
20
+ * Some methods are still exposed (ZSTDLIB_API),
21
+ * because it used to be the only way to invoke MT compression.
22
+ * Now, it's recommended to use ZSTD_compress_generic() instead.
23
+ * These methods will stop being exposed in a future version */
20
24
 
21
25
  /* === Dependencies === */
22
- #include <stddef.h> /* size_t */
26
+ #include <stddef.h> /* size_t */
23
27
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
24
- #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
28
+ #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
25
29
 
26
30
 
27
- /* === Simple one-pass functions === */
28
-
31
+ /* === Memory management === */
29
32
  typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
30
33
  ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
31
- ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
34
+ ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
35
+ ZSTD_customMem cMem);
36
+ ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
37
+
38
+ ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
39
+
40
+
41
+ /* === Simple buffer-to-butter one-pass function === */
42
+
43
+ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
44
+ void* dst, size_t dstCapacity,
45
+ const void* src, size_t srcSize,
46
+ int compressionLevel);
32
47
 
33
- ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
34
- void* dst, size_t dstCapacity,
35
- const void* src, size_t srcSize,
36
- int compressionLevel);
37
48
 
38
49
 
39
50
  /* === Streaming functions === */
40
51
 
41
52
  ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
42
- ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
53
+ ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */
43
54
 
44
55
  ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
45
56
 
@@ -49,26 +60,77 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
49
60
 
50
61
  /* === Advanced functions and parameters === */
51
62
 
52
- #ifndef ZSTDMT_SECTION_SIZE_MIN
53
- # define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
63
+ #ifndef ZSTDMT_JOBSIZE_MIN
64
+ # define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
54
65
  #endif
55
66
 
56
- ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
57
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
67
+ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
68
+ void* dst, size_t dstCapacity,
69
+ const void* src, size_t srcSize,
70
+ const ZSTD_CDict* cdict,
71
+ ZSTD_parameters const params,
72
+ unsigned overlapLog);
73
+
74
+ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
75
+ const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
76
+ ZSTD_parameters params,
77
+ unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
78
+
79
+ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
80
+ const ZSTD_CDict* cdict,
81
+ ZSTD_frameParameters fparams,
82
+ unsigned long long pledgedSrcSize); /* note : zero means empty */
58
83
 
59
- /* ZSDTMT_parameter :
84
+ /* ZSTDMT_parameter :
60
85
  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
61
86
  typedef enum {
62
- ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
63
- ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
64
- } ZSDTMT_parameter;
87
+ ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
88
+ ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
89
+ } ZSTDMT_parameter;
65
90
 
66
91
  /* ZSTDMT_setMTCtxParameter() :
67
92
  * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
68
- * The function must be called typically after ZSTD_createCCtx().
93
+ * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
69
94
  * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
70
95
  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
71
- ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
96
+ ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
97
+
98
+
99
+ /*! ZSTDMT_compressStream_generic() :
100
+ * Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream()
101
+ * depending on flush directive.
102
+ * @return : minimum amount of data still to be flushed
103
+ * 0 if fully flushed
104
+ * or an error code */
105
+ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
106
+ ZSTD_outBuffer* output,
107
+ ZSTD_inBuffer* input,
108
+ ZSTD_EndDirective endOp);
109
+
110
+
111
+ /* === Private definitions; never ever use directly === */
112
+
113
+ size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
114
+
115
+ /* ZSTDMT_CCtxParam_setNbThreads()
116
+ * Set nbThreads, and clamp it correctly,
117
+ * also reset jobSize and overlapLog */
118
+ size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads);
119
+
120
+ /* ZSTDMT_getNbThreads():
121
+ * @return nb threads currently active in mtctx.
122
+ * mtctx must be valid */
123
+ size_t ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx);
124
+
125
+ /*! ZSTDMT_initCStream_internal() :
126
+ * Private use only. Init streaming operation.
127
+ * expects params to be valid.
128
+ * must receive dict, or cdict, or none, but not both.
129
+ * @return : 0, or an error code */
130
+ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
131
+ const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
132
+ const ZSTD_CDict* cdict,
133
+ ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
72
134
 
73
135
 
74
136
  #if defined (__cplusplus)
@@ -32,41 +32,31 @@
32
32
  - Public forum : https://groups.google.com/forum/#!forum/lz4c
33
33
  ****************************************************************** */
34
34
 
35
- /* **************************************************************
36
- * Compiler specifics
37
- ****************************************************************/
38
- #ifdef _MSC_VER /* Visual Studio */
39
- # define FORCE_INLINE static __forceinline
40
- # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
41
- #else
42
- # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
43
- # ifdef __GNUC__
44
- # define FORCE_INLINE static inline __attribute__((always_inline))
45
- # else
46
- # define FORCE_INLINE static inline
47
- # endif
48
- # else
49
- # define FORCE_INLINE static
50
- # endif /* __STDC_VERSION__ */
51
- #endif
52
-
53
-
54
35
  /* **************************************************************
55
36
  * Dependencies
56
37
  ****************************************************************/
57
38
  #include <string.h> /* memcpy, memset */
58
39
  #include "bitstream.h" /* BIT_* */
40
+ #include "compiler.h"
59
41
  #include "fse.h" /* header compression */
60
42
  #define HUF_STATIC_LINKING_ONLY
61
43
  #include "huf.h"
44
+ #include "error_private.h"
62
45
 
63
46
 
64
47
  /* **************************************************************
65
48
  * Error Management
66
49
  ****************************************************************/
50
+ #define HUF_isError ERR_isError
67
51
  #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
68
52
 
69
53
 
54
+ /* **************************************************************
55
+ * Byte alignment for workSpace management
56
+ ****************************************************************/
57
+ #define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
58
+ #define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
59
+
70
60
  /*-***************************/
71
61
  /* generic DTableDesc */
72
62
  /*-***************************/
@@ -87,16 +77,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
87
77
 
88
78
  typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
89
79
 
90
- size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
80
+ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
91
81
  {
92
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
93
- U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
94
82
  U32 tableLog = 0;
95
83
  U32 nbSymbols = 0;
96
84
  size_t iSize;
97
85
  void* const dtPtr = DTable + 1;
98
86
  HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
99
87
 
88
+ U32* rankVal;
89
+ BYTE* huffWeight;
90
+ size_t spaceUsed32 = 0;
91
+
92
+ rankVal = (U32 *)workSpace + spaceUsed32;
93
+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
94
+ huffWeight = (BYTE *)((U32 *)workSpace + spaceUsed32);
95
+ spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
96
+
97
+ if ((spaceUsed32 << 2) > wkspSize)
98
+ return ERROR(tableLog_tooLarge);
99
+ workSpace = (U32 *)workSpace + spaceUsed32;
100
+ wkspSize -= (spaceUsed32 << 2);
101
+
100
102
  HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
101
103
  /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
102
104
 
@@ -135,6 +137,13 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
135
137
  return iSize;
136
138
  }
137
139
 
140
+ size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
141
+ {
142
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
143
+ return HUF_readDTableX2_wksp(DTable, src, srcSize,
144
+ workSpace, sizeof(workSpace));
145
+ }
146
+
138
147
 
139
148
  static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
140
149
  {
@@ -155,7 +164,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
155
164
  if (MEM_64bits()) \
156
165
  HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
157
166
 
158
- FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
167
+ HINT_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
159
168
  {
160
169
  BYTE* const pStart = p;
161
170
 
@@ -212,11 +221,13 @@ size_t HUF_decompress1X2_usingDTable(
212
221
  return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
213
222
  }
214
223
 
215
- size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
224
+ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
225
+ const void* cSrc, size_t cSrcSize,
226
+ void* workSpace, size_t wkspSize)
216
227
  {
217
228
  const BYTE* ip = (const BYTE*) cSrc;
218
229
 
219
- size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize);
230
+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
220
231
  if (HUF_isError(hSize)) return hSize;
221
232
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
222
233
  ip += hSize; cSrcSize -= hSize;
@@ -224,6 +235,15 @@ size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
224
235
  return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
225
236
  }
226
237
 
238
+
239
+ size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
240
+ const void* cSrc, size_t cSrcSize)
241
+ {
242
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
243
+ return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
244
+ workSpace, sizeof(workSpace));
245
+ }
246
+
227
247
  size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
228
248
  {
229
249
  HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
@@ -335,11 +355,14 @@ size_t HUF_decompress4X2_usingDTable(
335
355
  }
336
356
 
337
357
 
338
- size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
358
+ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
359
+ const void* cSrc, size_t cSrcSize,
360
+ void* workSpace, size_t wkspSize)
339
361
  {
340
362
  const BYTE* ip = (const BYTE*) cSrc;
341
363
 
342
- size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize);
364
+ size_t const hSize = HUF_readDTableX2_wksp (dctx, cSrc, cSrcSize,
365
+ workSpace, wkspSize);
343
366
  if (HUF_isError(hSize)) return hSize;
344
367
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
345
368
  ip += hSize; cSrcSize -= hSize;
@@ -347,6 +370,13 @@ size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
347
370
  return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
348
371
  }
349
372
 
373
+
374
+ size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
375
+ {
376
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
377
+ return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
378
+ workSpace, sizeof(workSpace));
379
+ }
350
380
  size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
351
381
  {
352
382
  HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
@@ -403,7 +433,8 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
403
433
  } }
404
434
  }
405
435
 
406
- typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
436
+ typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
437
+ typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
407
438
 
408
439
  static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
409
440
  const sortedSymbol_t* sortedList, const U32 sortedListSize,
@@ -447,20 +478,43 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
447
478
  }
448
479
  }
449
480
 
450
- size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
481
+ size_t HUF_readDTableX4_wksp(HUF_DTable* DTable, const void* src,
482
+ size_t srcSize, void* workSpace,
483
+ size_t wkspSize)
451
484
  {
452
- BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
453
- sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
454
- U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 };
455
- U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 };
456
- U32* const rankStart = rankStart0+1;
457
- rankVal_t rankVal;
458
485
  U32 tableLog, maxW, sizeOfSort, nbSymbols;
459
486
  DTableDesc dtd = HUF_getDTableDesc(DTable);
460
487
  U32 const maxTableLog = dtd.maxTableLog;
461
488
  size_t iSize;
462
489
  void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
463
490
  HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
491
+ U32 *rankStart;
492
+
493
+ rankValCol_t* rankVal;
494
+ U32* rankStats;
495
+ U32* rankStart0;
496
+ sortedSymbol_t* sortedSymbol;
497
+ BYTE* weightList;
498
+ size_t spaceUsed32 = 0;
499
+
500
+ rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32);
501
+ spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
502
+ rankStats = (U32 *)workSpace + spaceUsed32;
503
+ spaceUsed32 += HUF_TABLELOG_MAX + 1;
504
+ rankStart0 = (U32 *)workSpace + spaceUsed32;
505
+ spaceUsed32 += HUF_TABLELOG_MAX + 2;
506
+ sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t);
507
+ spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
508
+ weightList = (BYTE *)((U32 *)workSpace + spaceUsed32);
509
+ spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
510
+
511
+ if ((spaceUsed32 << 2) > wkspSize)
512
+ return ERROR(tableLog_tooLarge);
513
+ workSpace = (U32 *)workSpace + spaceUsed32;
514
+ wkspSize -= (spaceUsed32 << 2);
515
+
516
+ rankStart = rankStart0 + 1;
517
+ memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
464
518
 
465
519
  HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
466
520
  if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
@@ -527,6 +581,12 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
527
581
  return iSize;
528
582
  }
529
583
 
584
+ size_t HUF_readDTableX4(HUF_DTable* DTable, const void* src, size_t srcSize)
585
+ {
586
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
587
+ return HUF_readDTableX4_wksp(DTable, src, srcSize,
588
+ workSpace, sizeof(workSpace));
589
+ }
530
590
 
531
591
  static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
532
592
  {
@@ -545,7 +605,8 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
545
605
  if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
546
606
  BIT_skipBits(DStream, dt[val].nbBits);
547
607
  if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
548
- DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
608
+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
609
+ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
549
610
  } }
550
611
  return 1;
551
612
  }
@@ -562,7 +623,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
562
623
  if (MEM_64bits()) \
563
624
  ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
564
625
 
565
- FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
626
+ HINT_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
566
627
  {
567
628
  BYTE* const pStart = p;
568
629
 
@@ -626,11 +687,14 @@ size_t HUF_decompress1X4_usingDTable(
626
687
  return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
627
688
  }
628
689
 
629
- size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
690
+ size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
691
+ const void* cSrc, size_t cSrcSize,
692
+ void* workSpace, size_t wkspSize)
630
693
  {
631
694
  const BYTE* ip = (const BYTE*) cSrc;
632
695
 
633
- size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize);
696
+ size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize,
697
+ workSpace, wkspSize);
634
698
  if (HUF_isError(hSize)) return hSize;
635
699
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
636
700
  ip += hSize; cSrcSize -= hSize;
@@ -638,6 +702,15 @@ size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
638
702
  return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
639
703
  }
640
704
 
705
+
706
+ size_t HUF_decompress1X4_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
707
+ const void* cSrc, size_t cSrcSize)
708
+ {
709
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
710
+ return HUF_decompress1X4_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
711
+ workSpace, sizeof(workSpace));
712
+ }
713
+
641
714
  size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
642
715
  {
643
716
  HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
@@ -748,11 +821,14 @@ size_t HUF_decompress4X4_usingDTable(
748
821
  }
749
822
 
750
823
 
751
- size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
824
+ size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
825
+ const void* cSrc, size_t cSrcSize,
826
+ void* workSpace, size_t wkspSize)
752
827
  {
753
828
  const BYTE* ip = (const BYTE*) cSrc;
754
829
 
755
- size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize);
830
+ size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize,
831
+ workSpace, wkspSize);
756
832
  if (HUF_isError(hSize)) return hSize;
757
833
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
758
834
  ip += hSize; cSrcSize -= hSize;
@@ -760,6 +836,15 @@ size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
760
836
  return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
761
837
  }
762
838
 
839
+
840
+ size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
841
+ const void* cSrc, size_t cSrcSize)
842
+ {
843
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
844
+ return HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
845
+ workSpace, sizeof(workSpace));
846
+ }
847
+
763
848
  size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
764
849
  {
765
850
  HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
@@ -816,11 +901,11 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
816
901
  * Tells which decoder is likely to decode faster,
817
902
  * based on a set of pre-determined metrics.
818
903
  * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
819
- * Assumption : 0 < cSrcSize < dstSize <= 128 KB */
904
+ * Assumption : 0 < cSrcSize, dstSize <= 128 KB */
820
905
  U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
821
906
  {
822
907
  /* decoder timing evaluation */
823
- U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
908
+ U32 const Q = cSrcSize >= dstSize ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
824
909
  U32 const D256 = (U32)(dstSize >> 8);
825
910
  U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
826
911
  U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
@@ -861,19 +946,32 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
861
946
  }
862
947
  }
863
948
 
864
- size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
949
+ size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
950
+ {
951
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
952
+ return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
953
+ workSpace, sizeof(workSpace));
954
+ }
955
+
956
+
957
+ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
958
+ size_t dstSize, const void* cSrc,
959
+ size_t cSrcSize, void* workSpace,
960
+ size_t wkspSize)
865
961
  {
866
962
  /* validation checks */
867
963
  if (dstSize == 0) return ERROR(dstSize_tooSmall);
868
- if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
964
+ if (cSrcSize == 0) return ERROR(corruption_detected);
869
965
 
870
966
  { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
871
- return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
872
- HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
967
+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
968
+ HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
873
969
  }
874
970
  }
875
971
 
876
- size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
972
+ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
973
+ const void* cSrc, size_t cSrcSize,
974
+ void* workSpace, size_t wkspSize)
877
975
  {
878
976
  /* validation checks */
879
977
  if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -882,7 +980,17 @@ size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
882
980
  if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
883
981
 
884
982
  { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
885
- return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
886
- HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
983
+ return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc,
984
+ cSrcSize, workSpace, wkspSize):
985
+ HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
986
+ cSrcSize, workSpace, wkspSize);
887
987
  }
888
988
  }
989
+
990
+ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
991
+ const void* cSrc, size_t cSrcSize)
992
+ {
993
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
994
+ return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
995
+ workSpace, sizeof(workSpace));
996
+ }