extzstd 0.1.1 → 0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +5 -5
  2. data/HISTORY.ja.md +18 -0
  3. data/README.md +15 -50
  4. data/contrib/zstd/CONTRIBUTING.md +1 -1
  5. data/contrib/zstd/COPYING +339 -0
  6. data/contrib/zstd/Makefile +82 -51
  7. data/contrib/zstd/NEWS +92 -5
  8. data/contrib/zstd/README.md +50 -41
  9. data/contrib/zstd/appveyor.yml +164 -102
  10. data/contrib/zstd/circle.yml +10 -22
  11. data/contrib/zstd/lib/BUCK +31 -10
  12. data/contrib/zstd/lib/Makefile +57 -31
  13. data/contrib/zstd/lib/README.md +68 -37
  14. data/contrib/zstd/lib/common/bitstream.h +130 -76
  15. data/contrib/zstd/lib/common/compiler.h +86 -0
  16. data/contrib/zstd/lib/common/error_private.c +15 -11
  17. data/contrib/zstd/lib/common/error_private.h +8 -8
  18. data/contrib/zstd/lib/common/fse.h +19 -9
  19. data/contrib/zstd/lib/common/fse_decompress.c +3 -22
  20. data/contrib/zstd/lib/common/huf.h +68 -26
  21. data/contrib/zstd/lib/common/mem.h +23 -35
  22. data/contrib/zstd/lib/common/pool.c +123 -63
  23. data/contrib/zstd/lib/common/pool.h +19 -10
  24. data/contrib/zstd/lib/common/threading.c +11 -16
  25. data/contrib/zstd/lib/common/threading.h +52 -33
  26. data/contrib/zstd/lib/common/xxhash.c +28 -22
  27. data/contrib/zstd/lib/common/zstd_common.c +40 -27
  28. data/contrib/zstd/lib/common/zstd_errors.h +43 -34
  29. data/contrib/zstd/lib/common/zstd_internal.h +131 -123
  30. data/contrib/zstd/lib/compress/fse_compress.c +17 -33
  31. data/contrib/zstd/lib/compress/huf_compress.c +15 -9
  32. data/contrib/zstd/lib/compress/zstd_compress.c +2096 -2363
  33. data/contrib/zstd/lib/compress/zstd_compress_internal.h +462 -0
  34. data/contrib/zstd/lib/compress/zstd_double_fast.c +309 -0
  35. data/contrib/zstd/lib/compress/zstd_double_fast.h +29 -0
  36. data/contrib/zstd/lib/compress/zstd_fast.c +243 -0
  37. data/contrib/zstd/lib/compress/zstd_fast.h +31 -0
  38. data/contrib/zstd/lib/compress/zstd_lazy.c +765 -0
  39. data/contrib/zstd/lib/compress/zstd_lazy.h +39 -0
  40. data/contrib/zstd/lib/compress/zstd_ldm.c +707 -0
  41. data/contrib/zstd/lib/compress/zstd_ldm.h +68 -0
  42. data/contrib/zstd/lib/compress/zstd_opt.c +785 -0
  43. data/contrib/zstd/lib/compress/zstd_opt.h +19 -908
  44. data/contrib/zstd/lib/compress/zstdmt_compress.c +737 -327
  45. data/contrib/zstd/lib/compress/zstdmt_compress.h +88 -26
  46. data/contrib/zstd/lib/decompress/huf_decompress.c +158 -50
  47. data/contrib/zstd/lib/decompress/zstd_decompress.c +884 -699
  48. data/contrib/zstd/lib/deprecated/zbuff.h +5 -4
  49. data/contrib/zstd/lib/deprecated/zbuff_common.c +5 -5
  50. data/contrib/zstd/lib/deprecated/zbuff_compress.c +6 -4
  51. data/contrib/zstd/lib/deprecated/zbuff_decompress.c +5 -4
  52. data/contrib/zstd/lib/dictBuilder/cover.c +93 -77
  53. data/contrib/zstd/lib/dictBuilder/zdict.c +107 -92
  54. data/contrib/zstd/lib/dictBuilder/zdict.h +112 -102
  55. data/contrib/zstd/lib/legacy/zstd_legacy.h +9 -4
  56. data/contrib/zstd/lib/legacy/zstd_v01.c +7 -6
  57. data/contrib/zstd/lib/legacy/zstd_v01.h +5 -4
  58. data/contrib/zstd/lib/legacy/zstd_v02.c +27 -99
  59. data/contrib/zstd/lib/legacy/zstd_v02.h +5 -4
  60. data/contrib/zstd/lib/legacy/zstd_v03.c +26 -98
  61. data/contrib/zstd/lib/legacy/zstd_v03.h +5 -4
  62. data/contrib/zstd/lib/legacy/zstd_v04.c +22 -91
  63. data/contrib/zstd/lib/legacy/zstd_v04.h +5 -4
  64. data/contrib/zstd/lib/legacy/zstd_v05.c +23 -99
  65. data/contrib/zstd/lib/legacy/zstd_v05.h +5 -4
  66. data/contrib/zstd/lib/legacy/zstd_v06.c +22 -96
  67. data/contrib/zstd/lib/legacy/zstd_v06.h +5 -4
  68. data/contrib/zstd/lib/legacy/zstd_v07.c +19 -95
  69. data/contrib/zstd/lib/legacy/zstd_v07.h +5 -4
  70. data/contrib/zstd/lib/zstd.h +895 -271
  71. data/ext/extconf.rb +11 -2
  72. data/ext/extzstd.c +45 -128
  73. data/ext/extzstd.h +74 -31
  74. data/ext/extzstd_stream.c +401 -142
  75. data/ext/zstd_common.c +5 -0
  76. data/ext/zstd_compress.c +8 -0
  77. data/ext/zstd_decompress.c +1 -0
  78. data/ext/zstd_dictbuilder.c +2 -0
  79. data/lib/extzstd/version.rb +1 -1
  80. data/lib/extzstd.rb +48 -1
  81. data/test/test_basic.rb +9 -1
  82. metadata +17 -7
  83. data/HISTORY.ja +0 -10
  84. data/contrib/zstd/LICENSE-examples +0 -11
  85. data/contrib/zstd/PATENTS +0 -33
@@ -1,10 +1,11 @@
1
- /**
1
+ /*
2
2
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3
3
  * All rights reserved.
4
4
  *
5
- * This source code is licensed under the BSD-style license found in the
6
- * LICENSE file in the root directory of this source tree. An additional grant
7
- * of patent rights can be found in the PATENTS file in the same directory.
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
8
9
  */
9
10
 
10
11
  #ifndef ZSTDMT_COMPRESS_H
@@ -15,31 +16,41 @@
15
16
  #endif
16
17
 
17
18
 
18
- /* Note : All prototypes defined in this file shall be considered experimental.
19
- * There is no guarantee of API continuity (yet) on any of these prototypes */
19
+ /* Note : This is an internal API.
20
+ * Some methods are still exposed (ZSTDLIB_API),
21
+ * because it used to be the only way to invoke MT compression.
22
+ * Now, it's recommended to use ZSTD_compress_generic() instead.
23
+ * These methods will stop being exposed in a future version */
20
24
 
21
25
  /* === Dependencies === */
22
- #include <stddef.h> /* size_t */
26
+ #include <stddef.h> /* size_t */
23
27
  #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
24
- #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
28
+ #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
25
29
 
26
30
 
27
- /* === Simple one-pass functions === */
28
-
31
+ /* === Memory management === */
29
32
  typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
30
33
  ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
31
- ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
34
+ ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
35
+ ZSTD_customMem cMem);
36
+ ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
37
+
38
+ ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
39
+
40
+
41
+ /* === Simple buffer-to-butter one-pass function === */
42
+
43
+ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
44
+ void* dst, size_t dstCapacity,
45
+ const void* src, size_t srcSize,
46
+ int compressionLevel);
32
47
 
33
- ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
34
- void* dst, size_t dstCapacity,
35
- const void* src, size_t srcSize,
36
- int compressionLevel);
37
48
 
38
49
 
39
50
  /* === Streaming functions === */
40
51
 
41
52
  ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
42
- ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
53
+ ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */
43
54
 
44
55
  ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
45
56
 
@@ -49,26 +60,77 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
49
60
 
50
61
  /* === Advanced functions and parameters === */
51
62
 
52
- #ifndef ZSTDMT_SECTION_SIZE_MIN
53
- # define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
63
+ #ifndef ZSTDMT_JOBSIZE_MIN
64
+ # define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
54
65
  #endif
55
66
 
56
- ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */
57
- ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
67
+ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
68
+ void* dst, size_t dstCapacity,
69
+ const void* src, size_t srcSize,
70
+ const ZSTD_CDict* cdict,
71
+ ZSTD_parameters const params,
72
+ unsigned overlapLog);
73
+
74
+ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
75
+ const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
76
+ ZSTD_parameters params,
77
+ unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */
78
+
79
+ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
80
+ const ZSTD_CDict* cdict,
81
+ ZSTD_frameParameters fparams,
82
+ unsigned long long pledgedSrcSize); /* note : zero means empty */
58
83
 
59
- /* ZSDTMT_parameter :
84
+ /* ZSTDMT_parameter :
60
85
  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
61
86
  typedef enum {
62
- ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
63
- ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
64
- } ZSDTMT_parameter;
87
+ ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
88
+ ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
89
+ } ZSTDMT_parameter;
65
90
 
66
91
  /* ZSTDMT_setMTCtxParameter() :
67
92
  * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
68
- * The function must be called typically after ZSTD_createCCtx().
93
+ * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
69
94
  * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
70
95
  * @return : 0, or an error code (which can be tested using ZSTD_isError()) */
71
- ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
96
+ ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
97
+
98
+
99
+ /*! ZSTDMT_compressStream_generic() :
100
+ * Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream()
101
+ * depending on flush directive.
102
+ * @return : minimum amount of data still to be flushed
103
+ * 0 if fully flushed
104
+ * or an error code */
105
+ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
106
+ ZSTD_outBuffer* output,
107
+ ZSTD_inBuffer* input,
108
+ ZSTD_EndDirective endOp);
109
+
110
+
111
+ /* === Private definitions; never ever use directly === */
112
+
113
+ size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
114
+
115
+ /* ZSTDMT_CCtxParam_setNbThreads()
116
+ * Set nbThreads, and clamp it correctly,
117
+ * also reset jobSize and overlapLog */
118
+ size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads);
119
+
120
+ /* ZSTDMT_getNbThreads():
121
+ * @return nb threads currently active in mtctx.
122
+ * mtctx must be valid */
123
+ size_t ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx);
124
+
125
+ /*! ZSTDMT_initCStream_internal() :
126
+ * Private use only. Init streaming operation.
127
+ * expects params to be valid.
128
+ * must receive dict, or cdict, or none, but not both.
129
+ * @return : 0, or an error code */
130
+ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
131
+ const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
132
+ const ZSTD_CDict* cdict,
133
+ ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
72
134
 
73
135
 
74
136
  #if defined (__cplusplus)
@@ -32,41 +32,31 @@
32
32
  - Public forum : https://groups.google.com/forum/#!forum/lz4c
33
33
  ****************************************************************** */
34
34
 
35
- /* **************************************************************
36
- * Compiler specifics
37
- ****************************************************************/
38
- #ifdef _MSC_VER /* Visual Studio */
39
- # define FORCE_INLINE static __forceinline
40
- # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
41
- #else
42
- # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
43
- # ifdef __GNUC__
44
- # define FORCE_INLINE static inline __attribute__((always_inline))
45
- # else
46
- # define FORCE_INLINE static inline
47
- # endif
48
- # else
49
- # define FORCE_INLINE static
50
- # endif /* __STDC_VERSION__ */
51
- #endif
52
-
53
-
54
35
  /* **************************************************************
55
36
  * Dependencies
56
37
  ****************************************************************/
57
38
  #include <string.h> /* memcpy, memset */
58
39
  #include "bitstream.h" /* BIT_* */
40
+ #include "compiler.h"
59
41
  #include "fse.h" /* header compression */
60
42
  #define HUF_STATIC_LINKING_ONLY
61
43
  #include "huf.h"
44
+ #include "error_private.h"
62
45
 
63
46
 
64
47
  /* **************************************************************
65
48
  * Error Management
66
49
  ****************************************************************/
50
+ #define HUF_isError ERR_isError
67
51
  #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
68
52
 
69
53
 
54
+ /* **************************************************************
55
+ * Byte alignment for workSpace management
56
+ ****************************************************************/
57
+ #define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
58
+ #define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
59
+
70
60
  /*-***************************/
71
61
  /* generic DTableDesc */
72
62
  /*-***************************/
@@ -87,16 +77,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
87
77
 
88
78
  typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
89
79
 
90
- size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
80
+ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
91
81
  {
92
- BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
93
- U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
94
82
  U32 tableLog = 0;
95
83
  U32 nbSymbols = 0;
96
84
  size_t iSize;
97
85
  void* const dtPtr = DTable + 1;
98
86
  HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
99
87
 
88
+ U32* rankVal;
89
+ BYTE* huffWeight;
90
+ size_t spaceUsed32 = 0;
91
+
92
+ rankVal = (U32 *)workSpace + spaceUsed32;
93
+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
94
+ huffWeight = (BYTE *)((U32 *)workSpace + spaceUsed32);
95
+ spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
96
+
97
+ if ((spaceUsed32 << 2) > wkspSize)
98
+ return ERROR(tableLog_tooLarge);
99
+ workSpace = (U32 *)workSpace + spaceUsed32;
100
+ wkspSize -= (spaceUsed32 << 2);
101
+
100
102
  HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
101
103
  /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
102
104
 
@@ -135,6 +137,13 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
135
137
  return iSize;
136
138
  }
137
139
 
140
+ size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
141
+ {
142
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
143
+ return HUF_readDTableX2_wksp(DTable, src, srcSize,
144
+ workSpace, sizeof(workSpace));
145
+ }
146
+
138
147
 
139
148
  static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
140
149
  {
@@ -155,7 +164,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
155
164
  if (MEM_64bits()) \
156
165
  HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
157
166
 
158
- FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
167
+ HINT_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
159
168
  {
160
169
  BYTE* const pStart = p;
161
170
 
@@ -212,11 +221,13 @@ size_t HUF_decompress1X2_usingDTable(
212
221
  return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
213
222
  }
214
223
 
215
- size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
224
+ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
225
+ const void* cSrc, size_t cSrcSize,
226
+ void* workSpace, size_t wkspSize)
216
227
  {
217
228
  const BYTE* ip = (const BYTE*) cSrc;
218
229
 
219
- size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize);
230
+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
220
231
  if (HUF_isError(hSize)) return hSize;
221
232
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
222
233
  ip += hSize; cSrcSize -= hSize;
@@ -224,6 +235,15 @@ size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
224
235
  return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
225
236
  }
226
237
 
238
+
239
+ size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
240
+ const void* cSrc, size_t cSrcSize)
241
+ {
242
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
243
+ return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
244
+ workSpace, sizeof(workSpace));
245
+ }
246
+
227
247
  size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
228
248
  {
229
249
  HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
@@ -335,11 +355,14 @@ size_t HUF_decompress4X2_usingDTable(
335
355
  }
336
356
 
337
357
 
338
- size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
358
+ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
359
+ const void* cSrc, size_t cSrcSize,
360
+ void* workSpace, size_t wkspSize)
339
361
  {
340
362
  const BYTE* ip = (const BYTE*) cSrc;
341
363
 
342
- size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize);
364
+ size_t const hSize = HUF_readDTableX2_wksp (dctx, cSrc, cSrcSize,
365
+ workSpace, wkspSize);
343
366
  if (HUF_isError(hSize)) return hSize;
344
367
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
345
368
  ip += hSize; cSrcSize -= hSize;
@@ -347,6 +370,13 @@ size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
347
370
  return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
348
371
  }
349
372
 
373
+
374
+ size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
375
+ {
376
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
377
+ return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
378
+ workSpace, sizeof(workSpace));
379
+ }
350
380
  size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
351
381
  {
352
382
  HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
@@ -403,7 +433,8 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
403
433
  } }
404
434
  }
405
435
 
406
- typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
436
+ typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
437
+ typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
407
438
 
408
439
  static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
409
440
  const sortedSymbol_t* sortedList, const U32 sortedListSize,
@@ -447,20 +478,43 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
447
478
  }
448
479
  }
449
480
 
450
- size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
481
+ size_t HUF_readDTableX4_wksp(HUF_DTable* DTable, const void* src,
482
+ size_t srcSize, void* workSpace,
483
+ size_t wkspSize)
451
484
  {
452
- BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
453
- sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
454
- U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 };
455
- U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 };
456
- U32* const rankStart = rankStart0+1;
457
- rankVal_t rankVal;
458
485
  U32 tableLog, maxW, sizeOfSort, nbSymbols;
459
486
  DTableDesc dtd = HUF_getDTableDesc(DTable);
460
487
  U32 const maxTableLog = dtd.maxTableLog;
461
488
  size_t iSize;
462
489
  void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
463
490
  HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
491
+ U32 *rankStart;
492
+
493
+ rankValCol_t* rankVal;
494
+ U32* rankStats;
495
+ U32* rankStart0;
496
+ sortedSymbol_t* sortedSymbol;
497
+ BYTE* weightList;
498
+ size_t spaceUsed32 = 0;
499
+
500
+ rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32);
501
+ spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
502
+ rankStats = (U32 *)workSpace + spaceUsed32;
503
+ spaceUsed32 += HUF_TABLELOG_MAX + 1;
504
+ rankStart0 = (U32 *)workSpace + spaceUsed32;
505
+ spaceUsed32 += HUF_TABLELOG_MAX + 2;
506
+ sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t);
507
+ spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
508
+ weightList = (BYTE *)((U32 *)workSpace + spaceUsed32);
509
+ spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
510
+
511
+ if ((spaceUsed32 << 2) > wkspSize)
512
+ return ERROR(tableLog_tooLarge);
513
+ workSpace = (U32 *)workSpace + spaceUsed32;
514
+ wkspSize -= (spaceUsed32 << 2);
515
+
516
+ rankStart = rankStart0 + 1;
517
+ memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
464
518
 
465
519
  HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
466
520
  if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
@@ -527,6 +581,12 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
527
581
  return iSize;
528
582
  }
529
583
 
584
+ size_t HUF_readDTableX4(HUF_DTable* DTable, const void* src, size_t srcSize)
585
+ {
586
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
587
+ return HUF_readDTableX4_wksp(DTable, src, srcSize,
588
+ workSpace, sizeof(workSpace));
589
+ }
530
590
 
531
591
  static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
532
592
  {
@@ -545,7 +605,8 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
545
605
  if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
546
606
  BIT_skipBits(DStream, dt[val].nbBits);
547
607
  if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
548
- DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
608
+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
609
+ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
549
610
  } }
550
611
  return 1;
551
612
  }
@@ -562,7 +623,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
562
623
  if (MEM_64bits()) \
563
624
  ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
564
625
 
565
- FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
626
+ HINT_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
566
627
  {
567
628
  BYTE* const pStart = p;
568
629
 
@@ -626,11 +687,14 @@ size_t HUF_decompress1X4_usingDTable(
626
687
  return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
627
688
  }
628
689
 
629
- size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
690
+ size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
691
+ const void* cSrc, size_t cSrcSize,
692
+ void* workSpace, size_t wkspSize)
630
693
  {
631
694
  const BYTE* ip = (const BYTE*) cSrc;
632
695
 
633
- size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize);
696
+ size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize,
697
+ workSpace, wkspSize);
634
698
  if (HUF_isError(hSize)) return hSize;
635
699
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
636
700
  ip += hSize; cSrcSize -= hSize;
@@ -638,6 +702,15 @@ size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, cons
638
702
  return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
639
703
  }
640
704
 
705
+
706
+ size_t HUF_decompress1X4_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
707
+ const void* cSrc, size_t cSrcSize)
708
+ {
709
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
710
+ return HUF_decompress1X4_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
711
+ workSpace, sizeof(workSpace));
712
+ }
713
+
641
714
  size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
642
715
  {
643
716
  HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
@@ -748,11 +821,14 @@ size_t HUF_decompress4X4_usingDTable(
748
821
  }
749
822
 
750
823
 
751
- size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
824
+ size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
825
+ const void* cSrc, size_t cSrcSize,
826
+ void* workSpace, size_t wkspSize)
752
827
  {
753
828
  const BYTE* ip = (const BYTE*) cSrc;
754
829
 
755
- size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize);
830
+ size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize,
831
+ workSpace, wkspSize);
756
832
  if (HUF_isError(hSize)) return hSize;
757
833
  if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
758
834
  ip += hSize; cSrcSize -= hSize;
@@ -760,6 +836,15 @@ size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, cons
760
836
  return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
761
837
  }
762
838
 
839
+
840
+ size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
841
+ const void* cSrc, size_t cSrcSize)
842
+ {
843
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
844
+ return HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
845
+ workSpace, sizeof(workSpace));
846
+ }
847
+
763
848
  size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
764
849
  {
765
850
  HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
@@ -816,11 +901,11 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
816
901
  * Tells which decoder is likely to decode faster,
817
902
  * based on a set of pre-determined metrics.
818
903
  * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
819
- * Assumption : 0 < cSrcSize < dstSize <= 128 KB */
904
+ * Assumption : 0 < cSrcSize, dstSize <= 128 KB */
820
905
  U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
821
906
  {
822
907
  /* decoder timing evaluation */
823
- U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
908
+ U32 const Q = cSrcSize >= dstSize ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
824
909
  U32 const D256 = (U32)(dstSize >> 8);
825
910
  U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
826
911
  U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
@@ -861,19 +946,32 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
861
946
  }
862
947
  }
863
948
 
864
- size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
949
+ size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
950
+ {
951
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
952
+ return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
953
+ workSpace, sizeof(workSpace));
954
+ }
955
+
956
+
957
+ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
958
+ size_t dstSize, const void* cSrc,
959
+ size_t cSrcSize, void* workSpace,
960
+ size_t wkspSize)
865
961
  {
866
962
  /* validation checks */
867
963
  if (dstSize == 0) return ERROR(dstSize_tooSmall);
868
- if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
964
+ if (cSrcSize == 0) return ERROR(corruption_detected);
869
965
 
870
966
  { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
871
- return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
872
- HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
967
+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
968
+ HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
873
969
  }
874
970
  }
875
971
 
876
- size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
972
+ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
973
+ const void* cSrc, size_t cSrcSize,
974
+ void* workSpace, size_t wkspSize)
877
975
  {
878
976
  /* validation checks */
879
977
  if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -882,7 +980,17 @@ size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
882
980
  if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
883
981
 
884
982
  { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
885
- return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
886
- HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
983
+ return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc,
984
+ cSrcSize, workSpace, wkspSize):
985
+ HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
986
+ cSrcSize, workSpace, wkspSize);
887
987
  }
888
988
  }
989
+
990
+ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
991
+ const void* cSrc, size_t cSrcSize)
992
+ {
993
+ U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
994
+ return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
995
+ workSpace, sizeof(workSpace));
996
+ }