licentivo-launch 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +35 -0
  4. data/ext/launch_native/extconf.rb +28 -0
  5. data/lib/licentivo_launch/vendor/licentivo.rb +262 -0
  6. data/lib/licentivo_launch.rb +352 -0
  7. data/vendor/LICENSE +21 -0
  8. data/vendor/NOTICE +17 -0
  9. data/vendor/VERSION +1 -0
  10. data/vendor/c/app.c +33 -0
  11. data/vendor/c/demo.c +77 -0
  12. data/vendor/c/feature_tasks.h +72 -0
  13. data/vendor/c/licentivo.c +382 -0
  14. data/vendor/c/licentivo.h +67 -0
  15. data/vendor/c/quickstart.c +51 -0
  16. data/vendor/c/test_vectors.c +4 -0
  17. data/vendor/c/vendor/Monocypher-LICENCE.md +173 -0
  18. data/vendor/c/vendor/cJSON-LICENSE +20 -0
  19. data/vendor/c/vendor/cJSON.c +3191 -0
  20. data/vendor/c/vendor/cJSON.h +306 -0
  21. data/vendor/c/vendor/monocypher-ed25519.c +500 -0
  22. data/vendor/c/vendor/monocypher-ed25519.h +140 -0
  23. data/vendor/c/vendor/monocypher.c +2956 -0
  24. data/vendor/c/vendor/monocypher.h +321 -0
  25. data/vendor/curl/COPYING.txt +22 -0
  26. data/vendor/curl/bin/libcurl-x64.dll +0 -0
  27. data/vendor/curl/include/brotli/decode.h +409 -0
  28. data/vendor/curl/include/brotli/encode.h +505 -0
  29. data/vendor/curl/include/brotli/port.h +302 -0
  30. data/vendor/curl/include/brotli/shared_dictionary.h +100 -0
  31. data/vendor/curl/include/brotli/types.h +83 -0
  32. data/vendor/curl/include/curl/curl.h +3374 -0
  33. data/vendor/curl/include/curl/curlver.h +77 -0
  34. data/vendor/curl/include/curl/easy.h +123 -0
  35. data/vendor/curl/include/curl/header.h +74 -0
  36. data/vendor/curl/include/curl/mprintf.h +85 -0
  37. data/vendor/curl/include/curl/multi.h +561 -0
  38. data/vendor/curl/include/curl/options.h +70 -0
  39. data/vendor/curl/include/curl/stdcheaders.h +35 -0
  40. data/vendor/curl/include/curl/system.h +399 -0
  41. data/vendor/curl/include/curl/typecheck-gcc.h +964 -0
  42. data/vendor/curl/include/curl/urlapi.h +156 -0
  43. data/vendor/curl/include/curl/websockets.h +98 -0
  44. data/vendor/curl/include/libpsl.h +207 -0
  45. data/vendor/curl/include/libssh2.h +1516 -0
  46. data/vendor/curl/include/libssh2_publickey.h +128 -0
  47. data/vendor/curl/include/libssh2_sftp.h +382 -0
  48. data/vendor/curl/include/nghttp2/nghttp2.h +6878 -0
  49. data/vendor/curl/include/nghttp2/nghttp2ver.h +42 -0
  50. data/vendor/curl/include/nghttp3/nghttp3.h +3704 -0
  51. data/vendor/curl/include/nghttp3/version.h +46 -0
  52. data/vendor/curl/include/ngtcp2/ngtcp2.h +7485 -0
  53. data/vendor/curl/include/ngtcp2/ngtcp2_crypto.h +1071 -0
  54. data/vendor/curl/include/ngtcp2/ngtcp2_crypto_quictls.h +147 -0
  55. data/vendor/curl/include/ngtcp2/version.h +51 -0
  56. data/vendor/curl/include/openssl/aes.h +120 -0
  57. data/vendor/curl/include/openssl/asn1.h +1121 -0
  58. data/vendor/curl/include/openssl/asn1t.h +996 -0
  59. data/vendor/curl/include/openssl/bio.h +716 -0
  60. data/vendor/curl/include/openssl/blowfish.h +106 -0
  61. data/vendor/curl/include/openssl/bn.h +480 -0
  62. data/vendor/curl/include/openssl/buffer.h +102 -0
  63. data/vendor/curl/include/openssl/camellia.h +121 -0
  64. data/vendor/curl/include/openssl/cast.h +99 -0
  65. data/vendor/curl/include/openssl/chacha.h +54 -0
  66. data/vendor/curl/include/openssl/cmac.h +81 -0
  67. data/vendor/curl/include/openssl/cms.h +541 -0
  68. data/vendor/curl/include/openssl/comp.h +7 -0
  69. data/vendor/curl/include/openssl/conf.h +189 -0
  70. data/vendor/curl/include/openssl/crypto.h +458 -0
  71. data/vendor/curl/include/openssl/ct.h +567 -0
  72. data/vendor/curl/include/openssl/curve25519.h +104 -0
  73. data/vendor/curl/include/openssl/des.h +215 -0
  74. data/vendor/curl/include/openssl/dh.h +245 -0
  75. data/vendor/curl/include/openssl/dsa.h +263 -0
  76. data/vendor/curl/include/openssl/dtls1.h +107 -0
  77. data/vendor/curl/include/openssl/ec.h +675 -0
  78. data/vendor/curl/include/openssl/ecdh.h +6 -0
  79. data/vendor/curl/include/openssl/ecdsa.h +6 -0
  80. data/vendor/curl/include/openssl/engine.h +215 -0
  81. data/vendor/curl/include/openssl/err.h +325 -0
  82. data/vendor/curl/include/openssl/evp.h +1288 -0
  83. data/vendor/curl/include/openssl/hkdf.h +65 -0
  84. data/vendor/curl/include/openssl/hmac.h +101 -0
  85. data/vendor/curl/include/openssl/idea.h +99 -0
  86. data/vendor/curl/include/openssl/kdf.h +137 -0
  87. data/vendor/curl/include/openssl/lhash.h +179 -0
  88. data/vendor/curl/include/openssl/md4.h +105 -0
  89. data/vendor/curl/include/openssl/md5.h +99 -0
  90. data/vendor/curl/include/openssl/mlkem.h +236 -0
  91. data/vendor/curl/include/openssl/modes.h +118 -0
  92. data/vendor/curl/include/openssl/obj_mac.h +4646 -0
  93. data/vendor/curl/include/openssl/objects.h +137 -0
  94. data/vendor/curl/include/openssl/ocsp.h +484 -0
  95. data/vendor/curl/include/openssl/opensslconf.h +16 -0
  96. data/vendor/curl/include/openssl/opensslfeatures.h +153 -0
  97. data/vendor/curl/include/openssl/opensslv.h +18 -0
  98. data/vendor/curl/include/openssl/ossl_typ.h +196 -0
  99. data/vendor/curl/include/openssl/pem.h +542 -0
  100. data/vendor/curl/include/openssl/pkcs12.h +284 -0
  101. data/vendor/curl/include/openssl/pkcs7.h +523 -0
  102. data/vendor/curl/include/openssl/poly1305.h +45 -0
  103. data/vendor/curl/include/openssl/posix_time.h +54 -0
  104. data/vendor/curl/include/openssl/rand.h +118 -0
  105. data/vendor/curl/include/openssl/rc2.h +101 -0
  106. data/vendor/curl/include/openssl/rc4.h +88 -0
  107. data/vendor/curl/include/openssl/ripemd.h +107 -0
  108. data/vendor/curl/include/openssl/rsa.h +603 -0
  109. data/vendor/curl/include/openssl/safestack.h +1739 -0
  110. data/vendor/curl/include/openssl/sha.h +190 -0
  111. data/vendor/curl/include/openssl/sm3.h +49 -0
  112. data/vendor/curl/include/openssl/sm4.h +47 -0
  113. data/vendor/curl/include/openssl/srtp.h +148 -0
  114. data/vendor/curl/include/openssl/ssl.h +2336 -0
  115. data/vendor/curl/include/openssl/ssl3.h +441 -0
  116. data/vendor/curl/include/openssl/stack.h +99 -0
  117. data/vendor/curl/include/openssl/tls1.h +764 -0
  118. data/vendor/curl/include/openssl/ts.h +660 -0
  119. data/vendor/curl/include/openssl/txt_db.h +112 -0
  120. data/vendor/curl/include/openssl/ui.h +397 -0
  121. data/vendor/curl/include/openssl/x509.h +1037 -0
  122. data/vendor/curl/include/openssl/x509_vfy.h +464 -0
  123. data/vendor/curl/include/openssl/x509v3.h +1039 -0
  124. data/vendor/curl/include/zconf.h +206 -0
  125. data/vendor/curl/include/zdict.h +481 -0
  126. data/vendor/curl/include/zlib.h +1859 -0
  127. data/vendor/curl/include/zlib_name_mangling.h +8 -0
  128. data/vendor/curl/include/zstd.h +3198 -0
  129. data/vendor/curl/include/zstd_errors.h +107 -0
  130. data/vendor/curl/lib/libcurl.dll.a +0 -0
  131. data/vendor/curl/licenses/brotli/LICENSE.txt +19 -0
  132. data/vendor/curl/licenses/certdata/LICENSE.url +2 -0
  133. data/vendor/curl/licenses/libpsl/COPYING.txt +20 -0
  134. data/vendor/curl/licenses/libressl/COPYING.txt +133 -0
  135. data/vendor/curl/licenses/libssh2/COPYING.txt +43 -0
  136. data/vendor/curl/licenses/nghttp2/COPYING.txt +23 -0
  137. data/vendor/curl/licenses/nghttp3/COPYING.txt +22 -0
  138. data/vendor/curl/licenses/ngtcp2/COPYING.txt +22 -0
  139. data/vendor/curl/licenses/psl/LICENSE.txt +373 -0
  140. data/vendor/curl/licenses/trurl/COPYING.txt +22 -0
  141. data/vendor/curl/licenses/zlibng/LICENSE.md +19 -0
  142. data/vendor/curl/licenses/zstd/LICENSE.txt +30 -0
  143. metadata +205 -0
@@ -0,0 +1,206 @@
1
+ /* zconf.h -- configuration of the zlib compression library
2
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
3
+ * For conditions of distribution and use, see copyright notice in zlib.h
4
+ */
5
+
6
+ #ifndef ZCONF_H
7
+ #define ZCONF_H
8
+
9
+ #include "zlib_name_mangling.h"
10
+
11
+ #if !defined(_WIN32) && defined(__WIN32__)
12
+ # define _WIN32
13
+ #endif
14
+
15
+ /* Clang macro for detecting declspec support
16
+ * https://clang.llvm.org/docs/LanguageExtensions.html#has-declspec-attribute
17
+ */
18
+ #ifndef __has_declspec_attribute
19
+ # define __has_declspec_attribute(x) 0
20
+ #endif
21
+
22
+ #if defined(ZLIB_CONST) && !defined(z_const)
23
+ # define z_const const
24
+ #else
25
+ # define z_const
26
+ #endif
27
+
28
+ /* Maximum value for memLevel in deflateInit2 */
29
+ #ifndef MAX_MEM_LEVEL
30
+ # define MAX_MEM_LEVEL 9
31
+ #endif
32
+
33
+ /* Maximum value for windowBits in deflateInit2 and inflateInit2.
34
+ * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
35
+ * created by gzip. (Files created by minigzip can still be extracted by
36
+ * gzip.)
37
+ */
38
+ #ifndef MIN_WBITS
39
+ # define MIN_WBITS 8 /* 256 LZ77 window */
40
+ #endif
41
+ #ifndef MAX_WBITS
42
+ # define MAX_WBITS 15 /* 32K LZ77 window */
43
+ #endif
44
+
45
+ /* The memory requirements for deflate are (in bytes):
46
+ (1 << (windowBits+2)) + (1 << (memLevel+9))
47
+ that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values)
48
+ plus a few kilobytes for small objects. For example, if you want to reduce
49
+ the default memory requirements from 256K to 128K, compile with
50
+ make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
51
+ Of course this will generally degrade compression (there's no free lunch).
52
+
53
+ The memory requirements for inflate are (in bytes) 1 << windowBits
54
+ that is, 32K for windowBits=15 (default value) plus about 7 kilobytes
55
+ for small objects.
56
+ */
57
+
58
+ /* Type declarations */
59
+
60
+
61
+ #ifndef OF /* function prototypes */
62
+ # define OF(args) args
63
+ #endif
64
+
65
+ #ifdef ZLIB_INTERNAL
66
+ # define Z_INTERNAL ZLIB_INTERNAL
67
+ #endif
68
+
69
+ /* If building or using zlib as a DLL, define ZLIB_DLL.
70
+ * This is not mandatory, but it offers a little performance increase.
71
+ */
72
+ #if defined(ZLIB_DLL) && (defined(_WIN32) || (__has_declspec_attribute(dllexport) && __has_declspec_attribute(dllimport)))
73
+ # ifdef Z_INTERNAL
74
+ # define Z_EXTERN extern __declspec(dllexport)
75
+ # else
76
+ # define Z_EXTERN extern __declspec(dllimport)
77
+ # endif
78
+ #endif
79
+
80
+ /* If building or using zlib with the WINAPI/WINAPIV calling convention,
81
+ * define ZLIB_WINAPI.
82
+ * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI.
83
+ */
84
+ #if defined(ZLIB_WINAPI) && defined(_WIN32)
85
+ # ifndef WIN32_LEAN_AND_MEAN
86
+ # define WIN32_LEAN_AND_MEAN
87
+ # endif
88
+ # include <windows.h>
89
+ /* No need for _export, use ZLIB.DEF instead. */
90
+ /* For complete Windows compatibility, use WINAPI, not __stdcall. */
91
+ # define Z_EXPORT WINAPI
92
+ # define Z_EXPORTVA WINAPIV
93
+ #endif
94
+
95
+ #ifndef Z_EXTERN
96
+ # define Z_EXTERN extern
97
+ #endif
98
+ #ifndef Z_EXPORT
99
+ # define Z_EXPORT
100
+ #endif
101
+ #ifndef Z_EXPORTVA
102
+ # define Z_EXPORTVA
103
+ #endif
104
+
105
+ /* Conditional exports */
106
+ #define ZNG_CONDEXPORT Z_INTERNAL
107
+
108
+ /* For backwards compatibility */
109
+
110
+ #ifndef ZEXTERN
111
+ # define ZEXTERN Z_EXTERN
112
+ #endif
113
+ #ifndef ZEXPORT
114
+ # define ZEXPORT Z_EXPORT
115
+ #endif
116
+ #ifndef ZEXPORTVA
117
+ # define ZEXPORTVA Z_EXPORTVA
118
+ #endif
119
+ #ifndef FAR
120
+ # define FAR
121
+ #endif
122
+
123
+ /* Legacy zlib typedefs for backwards compatibility. Don't assume stdint.h is defined. */
124
+ typedef unsigned char Byte;
125
+ typedef Byte Bytef;
126
+
127
+ typedef unsigned int uInt; /* 16 bits or more */
128
+ typedef unsigned long uLong; /* 32 bits or more */
129
+
130
+ typedef char charf;
131
+ typedef int intf;
132
+ typedef uInt uIntf;
133
+ typedef uLong uLongf;
134
+
135
+ typedef void const *voidpc;
136
+ typedef void *voidpf;
137
+ typedef void *voidp;
138
+
139
+ typedef unsigned int z_crc_t;
140
+
141
+ #if 1 /* was set to #if 1 by configure/cmake/etc */
142
+ # define Z_HAVE_UNISTD_H
143
+ #endif
144
+
145
+ #ifdef NEED_PTRDIFF_T /* may be set to #if 1 by configure/cmake/etc */
146
+ typedef PTRDIFF_TYPE ptrdiff_t;
147
+ #endif
148
+
149
+ #include <sys/types.h> /* for off_t */
150
+
151
+ #include <stddef.h> /* for wchar_t and NULL */
152
+
153
+ /* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and
154
+ * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even
155
+ * though the former does not conform to the LFS document), but considering
156
+ * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as
157
+ * equivalently requesting no 64-bit operations
158
+ */
159
+ #if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1
160
+ # undef _LARGEFILE64_SOURCE
161
+ #endif
162
+
163
+ #if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
164
+ # include <unistd.h> /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
165
+ # ifndef z_off_t
166
+ # define z_off_t off_t
167
+ # endif
168
+ #endif
169
+
170
+ #if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0
171
+ # define Z_LFS64
172
+ #endif
173
+
174
+ #if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64)
175
+ # define Z_LARGE64
176
+ #endif
177
+
178
+ #if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64)
179
+ # define Z_WANT64
180
+ #endif
181
+
182
+ #if !defined(SEEK_SET)
183
+ # define SEEK_SET 0 /* Seek from beginning of file. */
184
+ # define SEEK_CUR 1 /* Seek from current position. */
185
+ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */
186
+ #endif
187
+
188
+ #ifndef z_off_t
189
+ # define z_off_t long
190
+ #endif
191
+
192
+ #if !defined(_WIN32) && defined(Z_LARGE64)
193
+ # define z_off64_t off64_t
194
+ #else
195
+ # if defined(__MSYS__)
196
+ # define z_off64_t _off64_t
197
+ # elif defined(_WIN32) && !defined(__GNUC__)
198
+ # define z_off64_t __int64
199
+ # else
200
+ # define z_off64_t z_off_t
201
+ # endif
202
+ #endif
203
+
204
+ typedef size_t z_size_t;
205
+
206
+ #endif /* ZCONF_H */
@@ -0,0 +1,481 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ * All rights reserved.
4
+ *
5
+ * This source code is licensed under both the BSD-style license (found in the
6
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7
+ * in the COPYING file in the root directory of this source tree).
8
+ * You may select, at your option, one of the above-listed licenses.
9
+ */
10
+
11
+ #ifndef ZSTD_ZDICT_H
12
+ #define ZSTD_ZDICT_H
13
+
14
+
15
+ /*====== Dependencies ======*/
16
+ #include <stddef.h> /* size_t */
17
+
18
+ #if defined (__cplusplus)
19
+ extern "C" {
20
+ #endif
21
+
22
+ /* ===== ZDICTLIB_API : control library symbols visibility ===== */
23
+ #ifndef ZDICTLIB_VISIBLE
24
+ /* Backwards compatibility with old macro name */
25
+ # ifdef ZDICTLIB_VISIBILITY
26
+ # define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY
27
+ # elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
28
+ # define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default")))
29
+ # else
30
+ # define ZDICTLIB_VISIBLE
31
+ # endif
32
+ #endif
33
+
34
+ #ifndef ZDICTLIB_HIDDEN
35
+ # if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
36
+ # define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden")))
37
+ # else
38
+ # define ZDICTLIB_HIDDEN
39
+ # endif
40
+ #endif
41
+
42
+ #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
43
+ # define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE
44
+ #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
45
+ # define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
46
+ #else
47
+ # define ZDICTLIB_API ZDICTLIB_VISIBLE
48
+ #endif
49
+
50
+ /*******************************************************************************
51
+ * Zstd dictionary builder
52
+ *
53
+ * FAQ
54
+ * ===
55
+ * Why should I use a dictionary?
56
+ * ------------------------------
57
+ *
58
+ * Zstd can use dictionaries to improve compression ratio of small data.
59
+ * Traditionally small files don't compress well because there is very little
60
+ * repetition in a single sample, since it is small. But, if you are compressing
61
+ * many similar files, like a bunch of JSON records that share the same
62
+ * structure, you can train a dictionary on ahead of time on some samples of
63
+ * these files. Then, zstd can use the dictionary to find repetitions that are
64
+ * present across samples. This can vastly improve compression ratio.
65
+ *
66
+ * When is a dictionary useful?
67
+ * ----------------------------
68
+ *
69
+ * Dictionaries are useful when compressing many small files that are similar.
70
+ * The larger a file is, the less benefit a dictionary will have. Generally,
71
+ * we don't expect dictionary compression to be effective past 100KB. And the
72
+ * smaller a file is, the more we would expect the dictionary to help.
73
+ *
74
+ * How do I use a dictionary?
75
+ * --------------------------
76
+ *
77
+ * Simply pass the dictionary to the zstd compressor with
78
+ * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to
79
+ * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other
80
+ * more advanced functions that allow selecting some options, see zstd.h for
81
+ * complete documentation.
82
+ *
83
+ * What is a zstd dictionary?
84
+ * --------------------------
85
+ *
86
+ * A zstd dictionary has two pieces: Its header, and its content. The header
87
+ * contains a magic number, the dictionary ID, and entropy tables. These
88
+ * entropy tables allow zstd to save on header costs in the compressed file,
89
+ * which really matters for small data. The content is just bytes, which are
90
+ * repeated content that is common across many samples.
91
+ *
92
+ * What is a raw content dictionary?
93
+ * ---------------------------------
94
+ *
95
+ * A raw content dictionary is just bytes. It doesn't have a zstd dictionary
96
+ * header, a dictionary ID, or entropy tables. Any buffer is a valid raw
97
+ * content dictionary.
98
+ *
99
+ * How do I train a dictionary?
100
+ * ----------------------------
101
+ *
102
+ * Gather samples from your use case. These samples should be similar to each
103
+ * other. If you have several use cases, you could try to train one dictionary
104
+ * per use case.
105
+ *
106
+ * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your
107
+ * dictionary. There are a few advanced versions of this function, but this
108
+ * is a great starting point. If you want to further tune your dictionary
109
+ * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow
110
+ * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`.
111
+ *
112
+ * If the dictionary training function fails, that is likely because you
113
+ * either passed too few samples, or a dictionary would not be effective
114
+ * for your data. Look at the messages that the dictionary trainer printed,
115
+ * if it doesn't say too few samples, then a dictionary would not be effective.
116
+ *
117
+ * How large should my dictionary be?
118
+ * ----------------------------------
119
+ *
120
+ * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB.
121
+ * The zstd CLI defaults to a 110KB dictionary. You likely don't need a
122
+ * dictionary larger than that. But, most use cases can get away with a
123
+ * smaller dictionary. The advanced dictionary builders can automatically
124
+ * shrink the dictionary for you, and select the smallest size that doesn't
125
+ * hurt compression ratio too much. See the `shrinkDict` parameter.
126
+ * A smaller dictionary can save memory, and potentially speed up
127
+ * compression.
128
+ *
129
+ * How many samples should I provide to the dictionary builder?
130
+ * ------------------------------------------------------------
131
+ *
132
+ * We generally recommend passing ~100x the size of the dictionary
133
+ * in samples. A few thousand should suffice. Having too few samples
134
+ * can hurt the dictionaries effectiveness. Having more samples will
135
+ * only improve the dictionaries effectiveness. But having too many
136
+ * samples can slow down the dictionary builder.
137
+ *
138
+ * How do I determine if a dictionary will be effective?
139
+ * -----------------------------------------------------
140
+ *
141
+ * Simply train a dictionary and try it out. You can use zstd's built in
142
+ * benchmarking tool to test the dictionary effectiveness.
143
+ *
144
+ * # Benchmark levels 1-3 without a dictionary
145
+ * zstd -b1e3 -r /path/to/my/files
146
+ * # Benchmark levels 1-3 with a dictionary
147
+ * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
148
+ *
149
+ * When should I retrain a dictionary?
150
+ * -----------------------------------
151
+ *
152
+ * You should retrain a dictionary when its effectiveness drops. Dictionary
153
+ * effectiveness drops as the data you are compressing changes. Generally, we do
154
+ * expect dictionaries to "decay" over time, as your data changes, but the rate
155
+ * at which they decay depends on your use case. Internally, we regularly
156
+ * retrain dictionaries, and if the new dictionary performs significantly
157
+ * better than the old dictionary, we will ship the new dictionary.
158
+ *
159
+ * I have a raw content dictionary, how do I turn it into a zstd dictionary?
160
+ * -------------------------------------------------------------------------
161
+ *
162
+ * If you have a raw content dictionary, e.g. by manually constructing it, or
163
+ * using a third-party dictionary builder, you can turn it into a zstd
164
+ * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to
165
+ * provide some samples of the data. It will add the zstd header to the
166
+ * raw content, which contains a dictionary ID and entropy tables, which
167
+ * will improve compression ratio, and allow zstd to write the dictionary ID
168
+ * into the frame, if you so choose.
169
+ *
170
+ * Do I have to use zstd's dictionary builder?
171
+ * -------------------------------------------
172
+ *
173
+ * No! You can construct dictionary content however you please, it is just
174
+ * bytes. It will always be valid as a raw content dictionary. If you want
175
+ * a zstd dictionary, which can improve compression ratio, use
176
+ * `ZDICT_finalizeDictionary()`.
177
+ *
178
+ * What is the attack surface of a zstd dictionary?
179
+ * ------------------------------------------------
180
+ *
181
+ * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so
182
+ * zstd should never crash, or access out-of-bounds memory no matter what
183
+ * the dictionary is. However, if an attacker can control the dictionary
184
+ * during decompression, they can cause zstd to generate arbitrary bytes,
185
+ * just like if they controlled the compressed data.
186
+ *
187
+ ******************************************************************************/
188
+
189
+
190
+ /*! ZDICT_trainFromBuffer():
191
+ * Train a dictionary from an array of samples.
192
+ * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
193
+ * f=20, and accel=1.
194
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
195
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
196
+ * The resulting dictionary will be saved into `dictBuffer`.
197
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
198
+ * or an error code, which can be tested with ZDICT_isError().
199
+ * Note: Dictionary training will fail if there are not enough samples to construct a
200
+ * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit).
201
+ * If dictionary training fails, you should use zstd without a dictionary, as the dictionary
202
+ * would've been ineffective anyways. If you believe your samples would benefit from a dictionary
203
+ * please open an issue with details, and we can look into it.
204
+ * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB.
205
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
206
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
207
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
208
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
209
+ */
210
+ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
211
+ const void* samplesBuffer,
212
+ const size_t* samplesSizes, unsigned nbSamples);
213
+
214
+ typedef struct {
215
+ int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */
216
+ unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
217
+ unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value)
218
+ * NOTE: The zstd format reserves some dictionary IDs for future use.
219
+ * You may use them in private settings, but be warned that they
220
+ * may be used by zstd in a public dictionary registry in the future.
221
+ * These dictionary IDs are:
222
+ * - low range : <= 32767
223
+ * - high range : >= (2^31)
224
+ */
225
+ } ZDICT_params_t;
226
+
227
+ /*! ZDICT_finalizeDictionary():
228
+ * Given a custom content as a basis for dictionary, and a set of samples,
229
+ * finalize dictionary by adding headers and statistics according to the zstd
230
+ * dictionary format.
231
+ *
232
+ * Samples must be stored concatenated in a flat buffer `samplesBuffer`,
233
+ * supplied with an array of sizes `samplesSizes`, providing the size of each
234
+ * sample in order. The samples are used to construct the statistics, so they
235
+ * should be representative of what you will compress with this dictionary.
236
+ *
237
+ * The compression level can be set in `parameters`. You should pass the
238
+ * compression level you expect to use in production. The statistics for each
239
+ * compression level differ, so tuning the dictionary for the compression level
240
+ * can help quite a bit.
241
+ *
242
+ * You can set an explicit dictionary ID in `parameters`, or allow us to pick
243
+ * a random dictionary ID for you, but we can't guarantee no collisions.
244
+ *
245
+ * The dstDictBuffer and the dictContent may overlap, and the content will be
246
+ * appended to the end of the header. If the header + the content doesn't fit in
247
+ * maxDictSize the beginning of the content is truncated to make room, since it
248
+ * is presumed that the most profitable content is at the end of the dictionary,
249
+ * since that is the cheapest to reference.
250
+ *
251
+ * `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN).
252
+ *
253
+ * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
254
+ * or an error code, which can be tested by ZDICT_isError().
255
+ * Note: ZDICT_finalizeDictionary() will push notifications into stderr if
256
+ * instructed to, using notificationLevel>0.
257
+ * NOTE: This function currently may fail in several edge cases including:
258
+ * * Not enough samples
259
+ * * Samples are uncompressible
260
+ * * Samples are all exactly the same
261
+ */
262
+ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
263
+ const void* dictContent, size_t dictContentSize,
264
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
265
+ ZDICT_params_t parameters);
266
+
267
+
268
+ /*====== Helper functions ======*/
269
+ ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */
270
+ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */
271
+ ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
272
+ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
273
+
274
+ #if defined (__cplusplus)
275
+ }
276
+ #endif
277
+
278
+ #endif /* ZSTD_ZDICT_H */
279
+
280
+ #if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
281
+ #define ZSTD_ZDICT_H_STATIC
282
+
283
+ #if defined (__cplusplus)
284
+ extern "C" {
285
+ #endif
286
+
287
+ /* This can be overridden externally to hide static symbols. */
288
+ #ifndef ZDICTLIB_STATIC_API
289
+ # if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
290
+ # define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE
291
+ # elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
292
+ # define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE
293
+ # else
294
+ # define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE
295
+ # endif
296
+ #endif
297
+
298
+ /* ====================================================================================
299
+ * The definitions in this section are considered experimental.
300
+ * They should never be used with a dynamic library, as they may change in the future.
301
+ * They are provided for advanced usages.
302
+ * Use them only in association with static linking.
303
+ * ==================================================================================== */
304
+
305
+ #define ZDICT_DICTSIZE_MIN 256
306
+ /* Deprecated: Remove in v1.6.0 */
307
+ #define ZDICT_CONTENTSIZE_MIN 128
308
+
309
+ /*! ZDICT_cover_params_t:
310
+ * k and d are the only required parameters.
311
+ * For others, value 0 means default.
312
+ */
313
+ typedef struct {
314
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
315
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
316
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
317
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
318
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
319
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
320
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
321
+ ZDICT_params_t zParams;
322
+ } ZDICT_cover_params_t;
323
+
324
+ typedef struct {
325
+ unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
326
+ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
327
+ unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/
328
+ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
329
+ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
330
+ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
331
+ unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
332
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
333
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
334
+
335
+ ZDICT_params_t zParams;
336
+ } ZDICT_fastCover_params_t;
337
+
338
+ /*! ZDICT_trainFromBuffer_cover():
339
+ * Train a dictionary from an array of samples using the COVER algorithm.
340
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
341
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
342
+ * The resulting dictionary will be saved into `dictBuffer`.
343
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
344
+ * or an error code, which can be tested with ZDICT_isError().
345
+ * See ZDICT_trainFromBuffer() for details on failure modes.
346
+ * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte.
347
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
348
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
349
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
350
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
351
+ */
352
+ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
353
+ void *dictBuffer, size_t dictBufferCapacity,
354
+ const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
355
+ ZDICT_cover_params_t parameters);
356
+
357
+ /*! ZDICT_optimizeTrainFromBuffer_cover():
358
+ * The same requirements as above hold for all the parameters except `parameters`.
359
+ * This function tries many parameter combinations and picks the best parameters.
360
+ * `*parameters` is filled with the best parameters found,
361
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
362
+ *
363
+ * All of the parameters d, k, steps are optional.
364
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
365
+ * if steps is zero it defaults to its default value.
366
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
367
+ *
368
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
369
+ * or an error code, which can be tested with ZDICT_isError().
370
+ * On success `*parameters` contains the parameters selected.
371
+ * See ZDICT_trainFromBuffer() for details on failure modes.
372
+ * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread.
373
+ */
374
+ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
375
+ void* dictBuffer, size_t dictBufferCapacity,
376
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
377
+ ZDICT_cover_params_t* parameters);
378
+
379
+ /*! ZDICT_trainFromBuffer_fastCover():
380
+ * Train a dictionary from an array of samples using a modified version of COVER algorithm.
381
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
382
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
383
+ * d and k are required.
384
+ * All other parameters are optional, will use default values if not provided
385
+ * The resulting dictionary will be saved into `dictBuffer`.
386
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
387
+ * or an error code, which can be tested with ZDICT_isError().
388
+ * See ZDICT_trainFromBuffer() for details on failure modes.
389
+ * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
390
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
391
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
392
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
393
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
394
+ */
395
+ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
396
+ size_t dictBufferCapacity, const void *samplesBuffer,
397
+ const size_t *samplesSizes, unsigned nbSamples,
398
+ ZDICT_fastCover_params_t parameters);
399
+
400
+ /*! ZDICT_optimizeTrainFromBuffer_fastCover():
401
+ * The same requirements as above hold for all the parameters except `parameters`.
402
+ * This function tries many parameter combinations (specifically, k and d combinations)
403
+ * and picks the best parameters. `*parameters` is filled with the best parameters found,
404
+ * dictionary constructed with those parameters is stored in `dictBuffer`.
405
+ * All of the parameters d, k, steps, f, and accel are optional.
406
+ * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
407
+ * if steps is zero it defaults to its default value.
408
+ * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
409
+ * If f is zero, default value of 20 is used.
410
+ * If accel is zero, default value of 1 is used.
411
+ *
412
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
413
+ * or an error code, which can be tested with ZDICT_isError().
414
+ * On success `*parameters` contains the parameters selected.
415
+ * See ZDICT_trainFromBuffer() for details on failure modes.
416
+ * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
417
+ */
418
+ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
419
+ size_t dictBufferCapacity, const void* samplesBuffer,
420
+ const size_t* samplesSizes, unsigned nbSamples,
421
+ ZDICT_fastCover_params_t* parameters);
422
+
423
+ typedef struct {
424
+ unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */
425
+ ZDICT_params_t zParams;
426
+ } ZDICT_legacy_params_t;
427
+
428
+ /*! ZDICT_trainFromBuffer_legacy():
429
+ * Train a dictionary from an array of samples.
430
+ * Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
431
+ * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
432
+ * The resulting dictionary will be saved into `dictBuffer`.
433
+ * `parameters` is optional and can be provided with values set to 0 to mean "default".
434
+ * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
435
+ * or an error code, which can be tested with ZDICT_isError().
436
+ * See ZDICT_trainFromBuffer() for details on failure modes.
437
+ * Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
438
+ * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
439
+ * In general, it's recommended to provide a few thousands samples, though this can vary a lot.
440
+ * It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
441
+ * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
442
+ */
443
+ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy(
444
+ void* dictBuffer, size_t dictBufferCapacity,
445
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
446
+ ZDICT_legacy_params_t parameters);
447
+
448
+
449
+ /* Deprecation warnings */
450
+ /* It is generally possible to disable deprecation warnings from compiler,
451
+ for example with -Wno-deprecated-declarations for gcc
452
+ or _CRT_SECURE_NO_WARNINGS in Visual.
453
+ Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */
454
+ #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS
455
+ # define ZDICT_DEPRECATED(message) /* disable deprecation warnings */
456
+ #else
457
+ # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
458
+ # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
459
+ # define ZDICT_DEPRECATED(message) [[deprecated(message)]]
460
+ # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405)
461
+ # define ZDICT_DEPRECATED(message) __attribute__((deprecated(message)))
462
+ # elif (ZDICT_GCC_VERSION >= 301)
463
+ # define ZDICT_DEPRECATED(message) __attribute__((deprecated))
464
+ # elif defined(_MSC_VER)
465
+ # define ZDICT_DEPRECATED(message) __declspec(deprecated(message))
466
+ # else
467
+ # pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler")
468
+ # define ZDICT_DEPRECATED(message)
469
+ # endif
470
+ #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */
471
+
472
+ ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead")
473
+ ZDICTLIB_STATIC_API
474
+ size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
475
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
476
+
477
+ #if defined (__cplusplus)
478
+ }
479
+ #endif
480
+
481
+ #endif /* ZSTD_ZDICT_H_STATIC */