zstdlib 0.6.0 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (129) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGES.md +20 -0
  3. data/README.md +7 -1
  4. data/Rakefile +38 -8
  5. data/ext/{zstdlib → zstdlib_c}/extconf.rb +10 -5
  6. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.2/zstdlib.c +2 -2
  7. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.3/zstdlib.c +2 -2
  8. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.4/zstdlib.c +2 -2
  9. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.5/zstdlib.c +2 -2
  10. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.6/zstdlib.c +2 -2
  11. data/ext/{zstdlib → zstdlib_c}/ruby/zlib-2.7/zstdlib.c +2 -2
  12. data/ext/zstdlib_c/ruby/zlib-3.0/zstdlib.c +4994 -0
  13. data/ext/zstdlib_c/ruby/zlib-3.1/zstdlib.c +5076 -0
  14. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/adler32.c +0 -0
  15. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/compress.c +0 -0
  16. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.c +0 -0
  17. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/crc32.h +0 -0
  18. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.c +0 -0
  19. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/deflate.h +0 -0
  20. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzclose.c +0 -0
  21. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzguts.h +0 -0
  22. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzlib.c +0 -0
  23. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzread.c +0 -0
  24. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/gzwrite.c +0 -0
  25. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/infback.c +0 -0
  26. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.c +0 -0
  27. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffast.h +0 -0
  28. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inffixed.h +0 -0
  29. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.c +0 -0
  30. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inflate.h +0 -0
  31. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.c +0 -0
  32. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/inftrees.h +0 -0
  33. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.c +0 -0
  34. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/trees.h +0 -0
  35. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/uncompr.c +0 -0
  36. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zconf.h +0 -0
  37. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zlib.h +0 -0
  38. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.c +0 -0
  39. data/ext/{zstdlib → zstdlib_c}/zlib-1.2.11/zutil.h +0 -0
  40. data/ext/{zstdlib → zstdlib_c}/zlib.mk +0 -0
  41. data/ext/{zstdlib → zstdlib_c}/zlibwrapper/zlibwrapper.c +1 -5
  42. data/ext/{zstdlib → zstdlib_c}/zlibwrapper.mk +0 -0
  43. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/bitstream.h +75 -57
  44. data/ext/zstdlib_c/zstd-1.5.2/lib/common/compiler.h +335 -0
  45. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/cpu.h +1 -3
  46. data/ext/zstdlib_c/zstd-1.5.2/lib/common/debug.c +24 -0
  47. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/debug.h +22 -49
  48. data/ext/zstdlib_c/zstd-1.5.2/lib/common/entropy_common.c +368 -0
  49. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/error_private.c +3 -1
  50. data/ext/zstdlib_c/zstd-1.5.2/lib/common/error_private.h +159 -0
  51. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse.h +51 -42
  52. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/fse_decompress.c +149 -57
  53. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/huf.h +60 -54
  54. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/mem.h +87 -98
  55. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.c +34 -23
  56. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/pool.h +5 -5
  57. data/ext/zstdlib_c/zstd-1.5.2/lib/common/portability_macros.h +137 -0
  58. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.c +10 -8
  59. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/threading.h +4 -3
  60. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.c +24 -0
  61. data/ext/zstdlib_c/zstd-1.5.2/lib/common/xxhash.h +5686 -0
  62. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/common/zstd_common.c +10 -10
  63. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_deps.h +111 -0
  64. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_internal.h +493 -0
  65. data/ext/zstdlib_c/zstd-1.5.2/lib/common/zstd_trace.h +163 -0
  66. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/clevels.h +134 -0
  67. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/fse_compress.c +105 -85
  68. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.c +41 -63
  69. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/hist.h +13 -33
  70. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/huf_compress.c +1370 -0
  71. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress.c +6327 -0
  72. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_internal.h +537 -82
  73. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.c +21 -16
  74. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_literals.h +4 -2
  75. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.c +61 -34
  76. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_compress_sequences.h +10 -3
  77. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.c +573 -0
  78. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_compress_superblock.h +32 -0
  79. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_cwksp.h +236 -95
  80. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.c +321 -143
  81. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_double_fast.h +2 -2
  82. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.c +328 -137
  83. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_fast.h +2 -2
  84. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.c +2104 -0
  85. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_lazy.h +125 -0
  86. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.c +336 -209
  87. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_ldm.h +15 -3
  88. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstd_ldm_geartab.h +106 -0
  89. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.c +439 -239
  90. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstd_opt.h +1 -1
  91. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/compress/zstdmt_compress.c +205 -462
  92. data/ext/zstdlib_c/zstd-1.5.2/lib/compress/zstdmt_compress.h +113 -0
  93. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress.c +1889 -0
  94. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/huf_decompress_amd64.S +585 -0
  95. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.c +20 -16
  96. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_ddict.h +3 -3
  97. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress.c +691 -230
  98. data/ext/zstdlib_c/zstd-1.5.2/lib/decompress/zstd_decompress_block.c +2072 -0
  99. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_block.h +16 -7
  100. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/decompress/zstd_decompress_internal.h +71 -10
  101. data/ext/zstdlib_c/zstd-1.5.2/lib/zdict.h +452 -0
  102. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/lib/zstd.h +760 -234
  103. data/ext/{zstdlib/zstd-1.4.4/lib/common → zstdlib_c/zstd-1.5.2/lib}/zstd_errors.h +3 -1
  104. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzclose.c +0 -0
  105. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzcompatibility.h +1 -1
  106. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzguts.h +0 -0
  107. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzlib.c +0 -0
  108. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzread.c +0 -0
  109. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/gzwrite.c +0 -0
  110. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.c +133 -44
  111. data/ext/{zstdlib/zstd-1.4.4 → zstdlib_c/zstd-1.5.2}/zlibWrapper/zstd_zlibwrapper.h +1 -1
  112. data/ext/zstdlib_c/zstd.mk +15 -0
  113. data/lib/zstdlib.rb +2 -2
  114. metadata +118 -107
  115. data/ext/zstdlib/zstd-1.4.4/lib/common/compiler.h +0 -159
  116. data/ext/zstdlib/zstd-1.4.4/lib/common/debug.c +0 -44
  117. data/ext/zstdlib/zstd-1.4.4/lib/common/entropy_common.c +0 -236
  118. data/ext/zstdlib/zstd-1.4.4/lib/common/error_private.h +0 -76
  119. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.c +0 -882
  120. data/ext/zstdlib/zstd-1.4.4/lib/common/xxhash.h +0 -305
  121. data/ext/zstdlib/zstd-1.4.4/lib/common/zstd_internal.h +0 -350
  122. data/ext/zstdlib/zstd-1.4.4/lib/compress/huf_compress.c +0 -798
  123. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_compress.c +0 -4103
  124. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.c +0 -1115
  125. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstd_lazy.h +0 -67
  126. data/ext/zstdlib/zstd-1.4.4/lib/compress/zstdmt_compress.h +0 -192
  127. data/ext/zstdlib/zstd-1.4.4/lib/decompress/huf_decompress.c +0 -1234
  128. data/ext/zstdlib/zstd-1.4.4/lib/decompress/zstd_decompress_block.c +0 -1323
  129. data/ext/zstdlib/zstd.mk +0 -14
@@ -0,0 +1,163 @@
1
+ /*
2
+ * Copyright (c) Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_TRACE_H
12
+ #define ZSTD_TRACE_H
13
+
14
+ #if defined (__cplusplus)
15
+ extern "C" {
16
+ #endif
17
+
18
+ #include <stddef.h>
19
+
20
+ /* weak symbol support
21
+ * For now, enable conservatively:
22
+ * - Only GNUC
23
+ * - Only ELF
24
+ * - Only x86-64 and i386
25
+ * Also, explicitly disable on platforms known not to work so they aren't
26
+ * forgotten in the future.
27
+ */
28
+ #if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
29
+ defined(__GNUC__) && defined(__ELF__) && \
30
+ (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) && \
31
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
32
+ !defined(__CYGWIN__) && !defined(_AIX)
33
+ # define ZSTD_HAVE_WEAK_SYMBOLS 1
34
+ #else
35
+ # define ZSTD_HAVE_WEAK_SYMBOLS 0
36
+ #endif
37
+ #if ZSTD_HAVE_WEAK_SYMBOLS
38
+ # define ZSTD_WEAK_ATTR __attribute__((__weak__))
39
+ #else
40
+ # define ZSTD_WEAK_ATTR
41
+ #endif
42
+
43
+ /* Only enable tracing when weak symbols are available. */
44
+ #ifndef ZSTD_TRACE
45
+ # define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
46
+ #endif
47
+
48
+ #if ZSTD_TRACE
49
+
50
+ struct ZSTD_CCtx_s;
51
+ struct ZSTD_DCtx_s;
52
+ struct ZSTD_CCtx_params_s;
53
+
54
+ typedef struct {
55
+ /**
56
+ * ZSTD_VERSION_NUMBER
57
+ *
58
+ * This is guaranteed to be the first member of ZSTD_trace.
59
+ * Otherwise, this struct is not stable between versions. If
60
+ * the version number does not match your expectation, you
61
+ * should not interpret the rest of the struct.
62
+ */
63
+ unsigned version;
64
+ /**
65
+ * Non-zero if streaming (de)compression is used.
66
+ */
67
+ unsigned streaming;
68
+ /**
69
+ * The dictionary ID.
70
+ */
71
+ unsigned dictionaryID;
72
+ /**
73
+ * Is the dictionary cold?
74
+ * Only set on decompression.
75
+ */
76
+ unsigned dictionaryIsCold;
77
+ /**
78
+ * The dictionary size or zero if no dictionary.
79
+ */
80
+ size_t dictionarySize;
81
+ /**
82
+ * The uncompressed size of the data.
83
+ */
84
+ size_t uncompressedSize;
85
+ /**
86
+ * The compressed size of the data.
87
+ */
88
+ size_t compressedSize;
89
+ /**
90
+ * The fully resolved CCtx parameters (NULL on decompression).
91
+ */
92
+ struct ZSTD_CCtx_params_s const* params;
93
+ /**
94
+ * The ZSTD_CCtx pointer (NULL on decompression).
95
+ */
96
+ struct ZSTD_CCtx_s const* cctx;
97
+ /**
98
+ * The ZSTD_DCtx pointer (NULL on compression).
99
+ */
100
+ struct ZSTD_DCtx_s const* dctx;
101
+ } ZSTD_Trace;
102
+
103
+ /**
104
+ * A tracing context. It must be 0 when tracing is disabled.
105
+ * Otherwise, any non-zero value returned by a tracing begin()
106
+ * function is presented to any subsequent calls to end().
107
+ *
108
+ * Any non-zero value is treated as tracing is enabled and not
109
+ * interpreted by the library.
110
+ *
111
+ * Two possible uses are:
112
+ * * A timestamp for when the begin() function was called.
113
+ * * A unique key identifying the (de)compression, like the
114
+ * address of the [dc]ctx pointer if you need to track
115
+ * more information than just a timestamp.
116
+ */
117
+ typedef unsigned long long ZSTD_TraceCtx;
118
+
119
+ /**
120
+ * Trace the beginning of a compression call.
121
+ * @param cctx The dctx pointer for the compression.
122
+ * It can be used as a key to map begin() to end().
123
+ * @returns Non-zero if tracing is enabled. The return value is
124
+ * passed to ZSTD_trace_compress_end().
125
+ */
126
+ ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
127
+ struct ZSTD_CCtx_s const* cctx);
128
+
129
+ /**
130
+ * Trace the end of a compression call.
131
+ * @param ctx The return value of ZSTD_trace_compress_begin().
132
+ * @param trace The zstd tracing info.
133
+ */
134
+ ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
135
+ ZSTD_TraceCtx ctx,
136
+ ZSTD_Trace const* trace);
137
+
138
+ /**
139
+ * Trace the beginning of a decompression call.
140
+ * @param dctx The dctx pointer for the decompression.
141
+ * It can be used as a key to map begin() to end().
142
+ * @returns Non-zero if tracing is enabled. The return value is
143
+ * passed to ZSTD_trace_compress_end().
144
+ */
145
+ ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
146
+ struct ZSTD_DCtx_s const* dctx);
147
+
148
+ /**
149
+ * Trace the end of a decompression call.
150
+ * @param ctx The return value of ZSTD_trace_decompress_begin().
151
+ * @param trace The zstd tracing info.
152
+ */
153
+ ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
154
+ ZSTD_TraceCtx ctx,
155
+ ZSTD_Trace const* trace);
156
+
157
+ #endif /* ZSTD_TRACE */
158
+
159
+ #if defined (__cplusplus)
160
+ }
161
+ #endif
162
+
163
+ #endif /* ZSTD_TRACE_H */
@@ -0,0 +1,134 @@
1
+ /*
2
+ * Copyright (c) Yann Collet, Facebook, Inc.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_CLEVELS_H
12
+ #define ZSTD_CLEVELS_H
13
+
14
+ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
15
+ #include "../zstd.h"
16
+
17
+ /*-===== Pre-defined compression levels =====-*/
18
+
19
+ #define ZSTD_MAX_CLEVEL 22
20
+
21
+ #ifdef __GNUC__
22
+ __attribute__((__unused__))
23
+ #endif
24
+
25
+ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
26
+ { /* "default" - for any srcSize > 256 KB */
27
+ /* W, C, H, S, L, TL, strat */
28
+ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */
29
+ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */
30
+ { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */
31
+ { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */
32
+ { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */
33
+ { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */
34
+ { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */
35
+ { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */
36
+ { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */
37
+ { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */
38
+ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */
39
+ { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */
40
+ { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */
41
+ { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */
42
+ { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */
43
+ { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */
44
+ { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */
45
+ { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */
46
+ { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */
47
+ { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */
48
+ { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */
49
+ { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */
50
+ { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */
51
+ },
52
+ { /* for srcSize <= 256 KB */
53
+ /* W, C, H, S, L, T, strat */
54
+ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
55
+ { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */
56
+ { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */
57
+ { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */
58
+ { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/
59
+ { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/
60
+ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/
61
+ { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */
62
+ { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
63
+ { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
64
+ { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
65
+ { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
66
+ { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
67
+ { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
68
+ { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
69
+ { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/
70
+ { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/
71
+ { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/
72
+ { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/
73
+ { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
74
+ { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/
75
+ { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/
76
+ { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/
77
+ },
78
+ { /* for srcSize <= 128 KB */
79
+ /* W, C, H, S, L, T, strat */
80
+ { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
81
+ { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */
82
+ { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
83
+ { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */
84
+ { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */
85
+ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
86
+ { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
87
+ { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */
88
+ { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
89
+ { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
90
+ { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
91
+ { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
92
+ { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
93
+ { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
94
+ { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/
95
+ { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/
96
+ { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/
97
+ { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/
98
+ { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/
99
+ { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/
100
+ { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/
101
+ { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
102
+ { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/
103
+ },
104
+ { /* for srcSize <= 16 KB */
105
+ /* W, C, H, S, L, T, strat */
106
+ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */
107
+ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */
108
+ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */
109
+ { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */
110
+ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */
111
+ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/
112
+ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */
113
+ { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */
114
+ { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/
115
+ { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/
116
+ { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/
117
+ { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/
118
+ { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/
119
+ { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/
120
+ { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/
121
+ { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/
122
+ { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/
123
+ { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/
124
+ { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/
125
+ { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/
126
+ { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/
127
+ { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/
128
+ { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/
129
+ },
130
+ };
131
+
132
+
133
+
134
+ #endif /* ZSTD_CLEVELS_H */
@@ -1,50 +1,31 @@
1
1
  /* ******************************************************************
2
- FSE : Finite State Entropy encoder
3
- Copyright (C) 2013-present, Yann Collet.
4
-
5
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
-
7
- Redistribution and use in source and binary forms, with or without
8
- modification, are permitted provided that the following conditions are
9
- met:
10
-
11
- * Redistributions of source code must retain the above copyright
12
- notice, this list of conditions and the following disclaimer.
13
- * Redistributions in binary form must reproduce the above
14
- copyright notice, this list of conditions and the following disclaimer
15
- in the documentation and/or other materials provided with the
16
- distribution.
17
-
18
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
-
30
- You can contact the author at :
31
- - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
32
- - Public forum : https://groups.google.com/forum/#!forum/lz4c
2
+ * FSE : Finite State Entropy encoder
3
+ * Copyright (c) Yann Collet, Facebook, Inc.
4
+ *
5
+ * You can contact the author at :
6
+ * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
7
+ * - Public forum : https://groups.google.com/forum/#!forum/lz4c
8
+ *
9
+ * This source code is licensed under both the BSD-style license (found in the
10
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
11
+ * in the COPYING file in the root directory of this source tree).
12
+ * You may select, at your option, one of the above-listed licenses.
33
13
  ****************************************************************** */
34
14
 
35
15
  /* **************************************************************
36
16
  * Includes
37
17
  ****************************************************************/
38
- #include <stdlib.h> /* malloc, free, qsort */
39
- #include <string.h> /* memcpy, memset */
40
- #include "compiler.h"
41
- #include "mem.h" /* U32, U16, etc. */
42
- #include "debug.h" /* assert, DEBUGLOG */
18
+ #include "../common/compiler.h"
19
+ #include "../common/mem.h" /* U32, U16, etc. */
20
+ #include "../common/debug.h" /* assert, DEBUGLOG */
43
21
  #include "hist.h" /* HIST_count_wksp */
44
- #include "bitstream.h"
22
+ #include "../common/bitstream.h"
45
23
  #define FSE_STATIC_LINKING_ONLY
46
- #include "fse.h"
47
- #include "error_private.h"
24
+ #include "../common/fse.h"
25
+ #include "../common/error_private.h"
26
+ #define ZSTD_DEPS_NEED_MALLOC
27
+ #define ZSTD_DEPS_NEED_MATH64
28
+ #include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
48
29
 
49
30
 
50
31
  /* **************************************************************
@@ -94,13 +75,16 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
94
75
  void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
95
76
  FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
96
77
  U32 const step = FSE_TABLESTEP(tableSize);
97
- U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
78
+ U32 const maxSV1 = maxSymbolValue+1;
79
+
80
+ U16* cumul = (U16*)workSpace; /* size = maxSV1 */
81
+ FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */
98
82
 
99
- FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace;
100
83
  U32 highThreshold = tableSize-1;
101
84
 
85
+ assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */
86
+ if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
102
87
  /* CTable header */
103
- if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
104
88
  tableU16[-2] = (U16) tableLog;
105
89
  tableU16[-1] = (U16) maxSymbolValue;
106
90
  assert(tableLog < 16); /* required for threshold strategy to work */
@@ -109,26 +93,67 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
109
93
  * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
110
94
 
111
95
  #ifdef __clang_analyzer__
112
- memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
96
+ ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
113
97
  #endif
114
98
 
115
99
  /* symbol start positions */
116
100
  { U32 u;
117
101
  cumul[0] = 0;
118
- for (u=1; u <= maxSymbolValue+1; u++) {
102
+ for (u=1; u <= maxSV1; u++) {
119
103
  if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
120
104
  cumul[u] = cumul[u-1] + 1;
121
105
  tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
122
106
  } else {
123
- cumul[u] = cumul[u-1] + normalizedCounter[u-1];
107
+ assert(normalizedCounter[u-1] >= 0);
108
+ cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1];
109
+ assert(cumul[u] >= cumul[u-1]); /* no overflow */
124
110
  } }
125
- cumul[maxSymbolValue+1] = tableSize+1;
111
+ cumul[maxSV1] = (U16)(tableSize+1);
126
112
  }
127
113
 
128
114
  /* Spread symbols */
129
- { U32 position = 0;
115
+ if (highThreshold == tableSize - 1) {
116
+ /* Case for no low prob count symbols. Lay down 8 bytes at a time
117
+ * to reduce branch misses since we are operating on a small block
118
+ */
119
+ BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */
120
+ { U64 const add = 0x0101010101010101ull;
121
+ size_t pos = 0;
122
+ U64 sv = 0;
123
+ U32 s;
124
+ for (s=0; s<maxSV1; ++s, sv += add) {
125
+ int i;
126
+ int const n = normalizedCounter[s];
127
+ MEM_write64(spread + pos, sv);
128
+ for (i = 8; i < n; i += 8) {
129
+ MEM_write64(spread + pos + i, sv);
130
+ }
131
+ assert(n>=0);
132
+ pos += (size_t)n;
133
+ }
134
+ }
135
+ /* Spread symbols across the table. Lack of lowprob symbols means that
136
+ * we don't need variable sized inner loop, so we can unroll the loop and
137
+ * reduce branch misses.
138
+ */
139
+ { size_t position = 0;
140
+ size_t s;
141
+ size_t const unroll = 2; /* Experimentally determined optimal unroll */
142
+ assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
143
+ for (s = 0; s < (size_t)tableSize; s += unroll) {
144
+ size_t u;
145
+ for (u = 0; u < unroll; ++u) {
146
+ size_t const uPosition = (position + (u * step)) & tableMask;
147
+ tableSymbol[uPosition] = spread[s + u];
148
+ }
149
+ position = (position + (unroll * step)) & tableMask;
150
+ }
151
+ assert(position == 0); /* Must have initialized all positions */
152
+ }
153
+ } else {
154
+ U32 position = 0;
130
155
  U32 symbol;
131
- for (symbol=0; symbol<=maxSymbolValue; symbol++) {
156
+ for (symbol=0; symbol<maxSV1; symbol++) {
132
157
  int nbOccurrences;
133
158
  int const freq = normalizedCounter[symbol];
134
159
  for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
@@ -137,7 +162,6 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
137
162
  while (position > highThreshold)
138
163
  position = (position + step) & tableMask; /* Low proba area */
139
164
  } }
140
-
141
165
  assert(position==0); /* Must have initialized all positions */
142
166
  }
143
167
 
@@ -161,16 +185,17 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
161
185
  case -1:
162
186
  case 1:
163
187
  symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
164
- symbolTT[s].deltaFindState = total - 1;
188
+ assert(total <= INT_MAX);
189
+ symbolTT[s].deltaFindState = (int)(total - 1);
165
190
  total ++;
166
191
  break;
167
192
  default :
168
- {
169
- U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
170
- U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
193
+ assert(normalizedCounter[s] > 1);
194
+ { U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1);
195
+ U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
171
196
  symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
172
- symbolTT[s].deltaFindState = total - normalizedCounter[s];
173
- total += normalizedCounter[s];
197
+ symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
198
+ total += (unsigned)normalizedCounter[s];
174
199
  } } } }
175
200
 
176
201
  #if 0 /* debug : symbol costs */
@@ -181,31 +206,26 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
181
206
  symbol, normalizedCounter[symbol],
182
207
  FSE_getMaxNbBits(symbolTT, symbol),
183
208
  (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
184
- }
185
- }
209
+ } }
186
210
  #endif
187
211
 
188
212
  return 0;
189
213
  }
190
214
 
191
215
 
192
- size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
193
- {
194
- FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
195
- return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
196
- }
197
-
198
-
199
216
 
200
217
  #ifndef FSE_COMMONDEFS_ONLY
201
218
 
202
-
203
219
  /*-**************************************************************
204
220
  * FSE NCount encoding
205
221
  ****************************************************************/
206
222
  size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
207
223
  {
208
- size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
224
+ size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog
225
+ + 4 /* bitCount initialized at 4 */
226
+ + 2 /* first two symbols may use one additional bit each */) / 8)
227
+ + 1 /* round up to whole nb bytes */
228
+ + 2 /* additional two bytes for bitstream flush */;
209
229
  return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
210
230
  }
211
231
 
@@ -327,10 +347,10 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
327
347
  size_t size;
328
348
  if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
329
349
  size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
330
- return (FSE_CTable*)malloc(size);
350
+ return (FSE_CTable*)ZSTD_malloc(size);
331
351
  }
332
352
 
333
- void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
353
+ void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
334
354
 
335
355
  /* provides the minimum logSize to safely represent a distribution */
336
356
  static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
@@ -361,11 +381,10 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
361
381
  return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
362
382
  }
363
383
 
364
-
365
384
  /* Secondary normalization method.
366
385
  To be used when primary method fails. */
367
386
 
368
- static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
387
+ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
369
388
  {
370
389
  short const NOT_YET_ASSIGNED = -2;
371
390
  U32 s;
@@ -382,7 +401,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
382
401
  continue;
383
402
  }
384
403
  if (count[s] <= lowThreshold) {
385
- norm[s] = -1;
404
+ norm[s] = lowProbCount;
386
405
  distributed++;
387
406
  total -= count[s];
388
407
  continue;
@@ -434,7 +453,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
434
453
 
435
454
  { U64 const vStepLog = 62 - tableLog;
436
455
  U64 const mid = (1ULL << (vStepLog-1)) - 1;
437
- U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
456
+ U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
438
457
  U64 tmpTotal = mid;
439
458
  for (s=0; s<=maxSymbolValue; s++) {
440
459
  if (norm[s]==NOT_YET_ASSIGNED) {
@@ -451,10 +470,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
451
470
  return 0;
452
471
  }
453
472
 
454
-
455
473
  size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
456
474
  const unsigned* count, size_t total,
457
- unsigned maxSymbolValue)
475
+ unsigned maxSymbolValue, unsigned useLowProbCount)
458
476
  {
459
477
  /* Sanity checks */
460
478
  if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
@@ -463,8 +481,9 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
463
481
  if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
464
482
 
465
483
  { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
484
+ short const lowProbCount = useLowProbCount ? -1 : 1;
466
485
  U64 const scale = 62 - tableLog;
467
- U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
486
+ U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
468
487
  U64 const vStep = 1ULL<<(scale-20);
469
488
  int stillToDistribute = 1<<tableLog;
470
489
  unsigned s;
@@ -476,7 +495,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
476
495
  if (count[s] == total) return 0; /* rle special case */
477
496
  if (count[s] == 0) { normalizedCounter[s]=0; continue; }
478
497
  if (count[s] <= lowThreshold) {
479
- normalizedCounter[s] = -1;
498
+ normalizedCounter[s] = lowProbCount;
480
499
  stillToDistribute--;
481
500
  } else {
482
501
  short proba = (short)((count[s]*step) >> scale);
@@ -490,7 +509,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
490
509
  } }
491
510
  if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
492
511
  /* corner case, need another normalization method */
493
- size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
512
+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
494
513
  if (FSE_isError(errorCode)) return errorCode;
495
514
  }
496
515
  else normalizedCounter[largest] += (short)stillToDistribute;
@@ -645,9 +664,7 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
645
664
 
646
665
  size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
647
666
 
648
- #define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
649
- #define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
650
-
667
+ #ifndef ZSTD_NO_UNUSED_FUNCTIONS
651
668
  /* FSE_compress_wksp() :
652
669
  * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
653
670
  * `wkspSize` size must be `(1<<tableLog)`.
@@ -666,7 +683,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
666
683
  size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
667
684
 
668
685
  /* init conditions */
669
- if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
686
+ if (wkspSize < FSE_COMPRESS_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
670
687
  if (srcSize <= 1) return 0; /* Not compressible */
671
688
  if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
672
689
  if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
@@ -679,7 +696,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
679
696
  }
680
697
 
681
698
  tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
682
- CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
699
+ CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) );
683
700
 
684
701
  /* Write table description header */
685
702
  { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
@@ -701,13 +718,16 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
701
718
 
702
719
  typedef struct {
703
720
  FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
704
- BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
721
+ union {
722
+ U32 hist_wksp[HIST_WKSP_SIZE_U32];
723
+ BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
724
+ } workspace;
705
725
  } fseWkspMax_t;
706
726
 
707
727
  size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
708
728
  {
709
729
  fseWkspMax_t scratchBuffer;
710
- DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
730
+ DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
711
731
  if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
712
732
  return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
713
733
  }
@@ -716,6 +736,6 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
716
736
  {
717
737
  return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
718
738
  }
719
-
739
+ #endif
720
740
 
721
741
  #endif /* FSE_COMMONDEFS_ONLY */