isomorfeus-ferret 0.12.7 → 0.13.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (164) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +101 -19
  3. data/README.md +54 -1
  4. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +22 -30
  5. data/ext/isomorfeus_ferret_ext/bm_hash.c +6 -12
  6. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +3 -6
  7. data/ext/isomorfeus_ferret_ext/bm_store.c +11 -22
  8. data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.c +1 -1
  9. data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +1 -1
  10. data/ext/isomorfeus_ferret_ext/bzip_blocksort.c +1094 -0
  11. data/ext/isomorfeus_ferret_ext/bzip_huffman.c +205 -0
  12. data/ext/isomorfeus_ferret_ext/bzlib.c +1572 -0
  13. data/ext/isomorfeus_ferret_ext/bzlib.h +282 -0
  14. data/ext/isomorfeus_ferret_ext/bzlib_compress.c +672 -0
  15. data/ext/isomorfeus_ferret_ext/bzlib_crctable.c +104 -0
  16. data/ext/isomorfeus_ferret_ext/bzlib_decompress.c +652 -0
  17. data/ext/isomorfeus_ferret_ext/bzlib_private.h +509 -0
  18. data/ext/isomorfeus_ferret_ext/bzlib_randtable.c +84 -0
  19. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -53
  20. data/ext/isomorfeus_ferret_ext/frb_analysis.c +785 -1192
  21. data/ext/isomorfeus_ferret_ext/frb_index.c +492 -474
  22. data/ext/isomorfeus_ferret_ext/frb_qparser.c +48 -60
  23. data/ext/isomorfeus_ferret_ext/frb_search.c +1520 -1002
  24. data/ext/isomorfeus_ferret_ext/frb_store.c +96 -96
  25. data/ext/isomorfeus_ferret_ext/frb_threading.h +0 -1
  26. data/ext/isomorfeus_ferret_ext/frb_utils.c +147 -196
  27. data/ext/isomorfeus_ferret_ext/frt_analysis.c +695 -1090
  28. data/ext/isomorfeus_ferret_ext/frt_analysis.h +174 -170
  29. data/ext/isomorfeus_ferret_ext/frt_array.c +2 -4
  30. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +9 -16
  31. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +32 -81
  32. data/ext/isomorfeus_ferret_ext/frt_document.c +15 -20
  33. data/ext/isomorfeus_ferret_ext/frt_document.h +10 -10
  34. data/ext/isomorfeus_ferret_ext/frt_except.c +5 -12
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +3 -3
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +6 -7
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +35 -46
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +1 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +105 -63
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +7 -3
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +1 -2
  42. data/ext/isomorfeus_ferret_ext/frt_ind.c +32 -35
  43. data/ext/isomorfeus_ferret_ext/frt_ind.h +9 -9
  44. data/ext/isomorfeus_ferret_ext/frt_index.c +580 -399
  45. data/ext/isomorfeus_ferret_ext/frt_index.h +272 -291
  46. data/ext/isomorfeus_ferret_ext/frt_mempool.c +1 -2
  47. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +4 -7
  48. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +67 -91
  49. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +35 -38
  50. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +53 -72
  51. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +25 -32
  52. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +21 -23
  53. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +66 -103
  54. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +207 -195
  55. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +20 -16
  56. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +17 -14
  57. data/ext/isomorfeus_ferret_ext/frt_q_range.c +102 -131
  58. data/ext/isomorfeus_ferret_ext/frt_q_span.c +179 -178
  59. data/ext/isomorfeus_ferret_ext/frt_q_term.c +47 -60
  60. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +18 -16
  61. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +45 -84
  62. data/ext/isomorfeus_ferret_ext/frt_search.c +105 -146
  63. data/ext/isomorfeus_ferret_ext/frt_search.h +331 -320
  64. data/ext/isomorfeus_ferret_ext/frt_similarity.c +5 -13
  65. data/ext/isomorfeus_ferret_ext/frt_similarity.h +7 -12
  66. data/ext/isomorfeus_ferret_ext/frt_sort.c +105 -149
  67. data/ext/isomorfeus_ferret_ext/frt_store.c +13 -7
  68. data/ext/isomorfeus_ferret_ext/frt_store.h +10 -2
  69. data/ext/isomorfeus_ferret_ext/frt_threading.h +0 -1
  70. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +21 -109
  71. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +2 -32
  72. data/ext/isomorfeus_ferret_ext/lz4.c +2495 -0
  73. data/ext/isomorfeus_ferret_ext/lz4.h +774 -0
  74. data/ext/isomorfeus_ferret_ext/lz4frame.c +1899 -0
  75. data/ext/isomorfeus_ferret_ext/lz4frame.h +623 -0
  76. data/ext/isomorfeus_ferret_ext/lz4hc.c +1615 -0
  77. data/ext/isomorfeus_ferret_ext/lz4hc.h +413 -0
  78. data/ext/isomorfeus_ferret_ext/lz4xxhash.c +1030 -0
  79. data/ext/isomorfeus_ferret_ext/lz4xxhash.h +328 -0
  80. data/ext/isomorfeus_ferret_ext/stem_modules.h +0 -86
  81. data/ext/isomorfeus_ferret_ext/test.c +1 -2
  82. data/ext/isomorfeus_ferret_ext/test_1710.c +11 -12
  83. data/ext/isomorfeus_ferret_ext/test_analysis.c +590 -583
  84. data/ext/isomorfeus_ferret_ext/test_compound_io.c +1 -1
  85. data/ext/isomorfeus_ferret_ext/test_document.c +19 -15
  86. data/ext/isomorfeus_ferret_ext/test_except.c +1 -2
  87. data/ext/isomorfeus_ferret_ext/test_fields.c +59 -60
  88. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +10 -27
  89. data/ext/isomorfeus_ferret_ext/test_filter.c +11 -8
  90. data/ext/isomorfeus_ferret_ext/test_hash.c +2 -2
  91. data/ext/isomorfeus_ferret_ext/test_hashset.c +1 -1
  92. data/ext/isomorfeus_ferret_ext/test_highlighter.c +15 -11
  93. data/ext/isomorfeus_ferret_ext/test_index.c +372 -365
  94. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +5 -3
  95. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +5 -3
  96. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +13 -10
  97. data/ext/isomorfeus_ferret_ext/test_q_parser.c +45 -7
  98. data/ext/isomorfeus_ferret_ext/test_q_span.c +15 -12
  99. data/ext/isomorfeus_ferret_ext/test_ram_store.c +3 -3
  100. data/ext/isomorfeus_ferret_ext/test_search.c +60 -62
  101. data/ext/isomorfeus_ferret_ext/test_segments.c +5 -4
  102. data/ext/isomorfeus_ferret_ext/test_sort.c +17 -14
  103. data/ext/isomorfeus_ferret_ext/test_store.c +2 -0
  104. data/ext/isomorfeus_ferret_ext/test_term.c +3 -1
  105. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -10
  106. data/ext/isomorfeus_ferret_ext/test_test.c +1 -2
  107. data/ext/isomorfeus_ferret_ext/test_threading.c +9 -10
  108. data/ext/isomorfeus_ferret_ext/testhelper.c +1 -2
  109. data/lib/isomorfeus/ferret/version.rb +1 -1
  110. metadata +27 -57
  111. data/ext/isomorfeus_ferret_ext/email.rl +0 -21
  112. data/ext/isomorfeus_ferret_ext/frt_scanner.c +0 -900
  113. data/ext/isomorfeus_ferret_ext/frt_scanner.h +0 -28
  114. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +0 -6706
  115. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +0 -4420
  116. data/ext/isomorfeus_ferret_ext/scanner.h +0 -28
  117. data/ext/isomorfeus_ferret_ext/scanner.in +0 -43
  118. data/ext/isomorfeus_ferret_ext/scanner.rl +0 -84
  119. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +0 -200
  120. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +0 -85
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.c +0 -1167
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.h +0 -6
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.c +0 -1433
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.h +0 -6
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +0 -301
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +0 -6
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +0 -590
  128. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +0 -6
  129. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +0 -1049
  130. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +0 -6
  131. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +0 -705
  132. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +0 -6
  133. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +0 -1239
  134. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +0 -6
  135. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +0 -477
  136. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +0 -6
  137. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +0 -1217
  138. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +0 -7
  139. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.c +0 -394
  140. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.h +0 -6
  141. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.c +0 -457
  142. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.h +0 -6
  143. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +0 -1009
  144. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +0 -6
  145. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +0 -259
  146. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +0 -6
  147. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +0 -704
  148. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +0 -6
  149. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +0 -948
  150. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +0 -6
  151. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +0 -1028
  152. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +0 -6
  153. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +0 -275
  154. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +0 -6
  155. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.c +0 -849
  156. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.h +0 -6
  157. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +0 -952
  158. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +0 -6
  159. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +0 -669
  160. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +0 -6
  161. data/ext/isomorfeus_ferret_ext/stem_modules.txt +0 -63
  162. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +0 -1854
  163. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +0 -1999
  164. data/ext/isomorfeus_ferret_ext/url.rl +0 -27
@@ -0,0 +1,509 @@
1
+
2
+ /*-------------------------------------------------------------*/
3
+ /*--- Private header file for the library. ---*/
4
+ /*--- bzlib_private.h ---*/
5
+ /*-------------------------------------------------------------*/
6
+
7
+ /* ------------------------------------------------------------------
8
+ This file is part of bzip2/libbzip2, a program and library for
9
+ lossless, block-sorting data compression.
10
+
11
+ bzip2/libbzip2 version 1.0.8 of 13 July 2019
12
+ Copyright (C) 1996-2019 Julian Seward <jseward@acm.org>
13
+
14
+ Please read the WARNING, DISCLAIMER and PATENTS sections in the
15
+ README file.
16
+
17
+ This program is released under the terms of the license contained
18
+ in the file LICENSE.
19
+ ------------------------------------------------------------------ */
20
+
21
+
22
+ #ifndef _BZLIB_PRIVATE_H
23
+ #define _BZLIB_PRIVATE_H
24
+
25
+ #include <stdlib.h>
26
+
27
+ #ifndef BZ_NO_STDIO
28
+ #include <stdio.h>
29
+ #include <ctype.h>
30
+ #include <string.h>
31
+ #endif
32
+
33
+ #include "bzlib.h"
34
+
35
+
36
+
37
+ /*-- General stuff. --*/
38
+
39
+ #define BZ_VERSION "1.0.8, 13-Jul-2019"
40
+
41
+ typedef char Char;
42
+ typedef unsigned char Bool;
43
+ typedef unsigned char UChar;
44
+ typedef int Int32;
45
+ typedef unsigned int UInt32;
46
+ typedef short Int16;
47
+ typedef unsigned short UInt16;
48
+
49
+ #define True ((Bool)1)
50
+ #define False ((Bool)0)
51
+
52
+ #ifndef __GNUC__
53
+ #define __inline__ /* */
54
+ #endif
55
+
56
+ #ifndef BZ_NO_STDIO
57
+
58
+ extern void BZ2_bz__AssertH__fail ( int errcode );
59
+ #define AssertH(cond,errcode) \
60
+ { if (!(cond)) BZ2_bz__AssertH__fail ( errcode ); }
61
+
62
+ #if BZ_DEBUG
63
+ #define AssertD(cond,msg) \
64
+ { if (!(cond)) { \
65
+ fprintf ( stderr, \
66
+ "\n\nlibbzip2(debug build): internal error\n\t%s\n", msg );\
67
+ exit(1); \
68
+ }}
69
+ #else
70
+ #define AssertD(cond,msg) /* */
71
+ #endif
72
+
73
+ #define VPrintf0(zf) \
74
+ fprintf(stderr,zf)
75
+ #define VPrintf1(zf,za1) \
76
+ fprintf(stderr,zf,za1)
77
+ #define VPrintf2(zf,za1,za2) \
78
+ fprintf(stderr,zf,za1,za2)
79
+ #define VPrintf3(zf,za1,za2,za3) \
80
+ fprintf(stderr,zf,za1,za2,za3)
81
+ #define VPrintf4(zf,za1,za2,za3,za4) \
82
+ fprintf(stderr,zf,za1,za2,za3,za4)
83
+ #define VPrintf5(zf,za1,za2,za3,za4,za5) \
84
+ fprintf(stderr,zf,za1,za2,za3,za4,za5)
85
+
86
+ #else
87
+
88
+ extern void bz_internal_error ( int errcode );
89
+ #define AssertH(cond,errcode) \
90
+ { if (!(cond)) bz_internal_error ( errcode ); }
91
+ #define AssertD(cond,msg) do { } while (0)
92
+ #define VPrintf0(zf) do { } while (0)
93
+ #define VPrintf1(zf,za1) do { } while (0)
94
+ #define VPrintf2(zf,za1,za2) do { } while (0)
95
+ #define VPrintf3(zf,za1,za2,za3) do { } while (0)
96
+ #define VPrintf4(zf,za1,za2,za3,za4) do { } while (0)
97
+ #define VPrintf5(zf,za1,za2,za3,za4,za5) do { } while (0)
98
+
99
+ #endif
100
+
101
+
102
+ #define BZALLOC(nnn) (strm->bzalloc)(strm->opaque,(nnn),1)
103
+ #define BZFREE(ppp) (strm->bzfree)(strm->opaque,(ppp))
104
+
105
+
106
+ /*-- Header bytes. --*/
107
+
108
+ #define BZ_HDR_B 0x42 /* 'B' */
109
+ #define BZ_HDR_Z 0x5a /* 'Z' */
110
+ #define BZ_HDR_h 0x68 /* 'h' */
111
+ #define BZ_HDR_0 0x30 /* '0' */
112
+
113
+ /*-- Constants for the back end. --*/
114
+
115
+ #define BZ_MAX_ALPHA_SIZE 258
116
+ #define BZ_MAX_CODE_LEN 23
117
+
118
+ #define BZ_RUNA 0
119
+ #define BZ_RUNB 1
120
+
121
+ #define BZ_N_GROUPS 6
122
+ #define BZ_G_SIZE 50
123
+ #define BZ_N_ITERS 4
124
+
125
+ #define BZ_MAX_SELECTORS (2 + (900000 / BZ_G_SIZE))
126
+
127
+
128
+
129
+ /*-- Stuff for randomising repetitive blocks. --*/
130
+
131
+ extern Int32 BZ2_rNums[512];
132
+
133
+ #define BZ_RAND_DECLS \
134
+ Int32 rNToGo; \
135
+ Int32 rTPos \
136
+
137
+ #define BZ_RAND_INIT_MASK \
138
+ s->rNToGo = 0; \
139
+ s->rTPos = 0 \
140
+
141
+ #define BZ_RAND_MASK ((s->rNToGo == 1) ? 1 : 0)
142
+
143
+ #define BZ_RAND_UPD_MASK \
144
+ if (s->rNToGo == 0) { \
145
+ s->rNToGo = BZ2_rNums[s->rTPos]; \
146
+ s->rTPos++; \
147
+ if (s->rTPos == 512) s->rTPos = 0; \
148
+ } \
149
+ s->rNToGo--;
150
+
151
+
152
+
153
+ /*-- Stuff for doing CRCs. --*/
154
+
155
+ extern UInt32 BZ2_crc32Table[256];
156
+
157
+ #define BZ_INITIALISE_CRC(crcVar) \
158
+ { \
159
+ crcVar = 0xffffffffL; \
160
+ }
161
+
162
+ #define BZ_FINALISE_CRC(crcVar) \
163
+ { \
164
+ crcVar = ~(crcVar); \
165
+ }
166
+
167
+ #define BZ_UPDATE_CRC(crcVar,cha) \
168
+ { \
169
+ crcVar = (crcVar << 8) ^ \
170
+ BZ2_crc32Table[(crcVar >> 24) ^ \
171
+ ((UChar)cha)]; \
172
+ }
173
+
174
+
175
+
176
+ /*-- States and modes for compression. --*/
177
+
178
+ #define BZ_M_IDLE 1
179
+ #define BZ_M_RUNNING 2
180
+ #define BZ_M_FLUSHING 3
181
+ #define BZ_M_FINISHING 4
182
+
183
+ #define BZ_S_OUTPUT 1
184
+ #define BZ_S_INPUT 2
185
+
186
+ #define BZ_N_RADIX 2
187
+ #define BZ_N_QSORT 12
188
+ #define BZ_N_SHELL 18
189
+ #define BZ_N_OVERSHOOT (BZ_N_RADIX + BZ_N_QSORT + BZ_N_SHELL + 2)
190
+
191
+
192
+
193
+
194
+ /*-- Structure holding all the compression-side stuff. --*/
195
+
196
+ typedef
197
+ struct {
198
+ /* pointer back to the struct bz_stream */
199
+ bz_stream* strm;
200
+
201
+ /* mode this stream is in, and whether inputting */
202
+ /* or outputting data */
203
+ Int32 mode;
204
+ Int32 state;
205
+
206
+ /* remembers avail_in when flush/finish requested */
207
+ UInt32 avail_in_expect;
208
+
209
+ /* for doing the block sorting */
210
+ UInt32* arr1;
211
+ UInt32* arr2;
212
+ UInt32* ftab;
213
+ Int32 origPtr;
214
+
215
+ /* aliases for arr1 and arr2 */
216
+ UInt32* ptr;
217
+ UChar* block;
218
+ UInt16* mtfv;
219
+ UChar* zbits;
220
+
221
+ /* for deciding when to use the fallback sorting algorithm */
222
+ Int32 workFactor;
223
+
224
+ /* run-length-encoding of the input */
225
+ UInt32 state_in_ch;
226
+ Int32 state_in_len;
227
+ BZ_RAND_DECLS;
228
+
229
+ /* input and output limits and current posns */
230
+ Int32 nblock;
231
+ Int32 nblockMAX;
232
+ Int32 numZ;
233
+ Int32 state_out_pos;
234
+
235
+ /* map of bytes used in block */
236
+ Int32 nInUse;
237
+ Bool inUse[256];
238
+ UChar unseqToSeq[256];
239
+
240
+ /* the buffer for bit stream creation */
241
+ UInt32 bsBuff;
242
+ Int32 bsLive;
243
+
244
+ /* block and combined CRCs */
245
+ UInt32 blockCRC;
246
+ UInt32 combinedCRC;
247
+
248
+ /* misc administratium */
249
+ Int32 verbosity;
250
+ Int32 blockNo;
251
+ Int32 blockSize100k;
252
+
253
+ /* stuff for coding the MTF values */
254
+ Int32 nMTF;
255
+ Int32 mtfFreq [BZ_MAX_ALPHA_SIZE];
256
+ UChar selector [BZ_MAX_SELECTORS];
257
+ UChar selectorMtf[BZ_MAX_SELECTORS];
258
+
259
+ UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
260
+ Int32 code [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
261
+ Int32 rfreq [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
262
+ /* second dimension: only 3 needed; 4 makes index calculations faster */
263
+ UInt32 len_pack[BZ_MAX_ALPHA_SIZE][4];
264
+
265
+ }
266
+ EState;
267
+
268
+
269
+
270
+ /*-- externs for compression. --*/
271
+
272
+ extern void
273
+ BZ2_blockSort ( EState* );
274
+
275
+ extern void
276
+ BZ2_compressBlock ( EState*, Bool );
277
+
278
+ extern void
279
+ BZ2_bsInitWrite ( EState* );
280
+
281
+ extern void
282
+ BZ2_hbAssignCodes ( Int32*, UChar*, Int32, Int32, Int32 );
283
+
284
+ extern void
285
+ BZ2_hbMakeCodeLengths ( UChar*, Int32*, Int32, Int32 );
286
+
287
+
288
+
289
+ /*-- states for decompression. --*/
290
+
291
+ #define BZ_X_IDLE 1
292
+ #define BZ_X_OUTPUT 2
293
+
294
+ #define BZ_X_MAGIC_1 10
295
+ #define BZ_X_MAGIC_2 11
296
+ #define BZ_X_MAGIC_3 12
297
+ #define BZ_X_MAGIC_4 13
298
+ #define BZ_X_BLKHDR_1 14
299
+ #define BZ_X_BLKHDR_2 15
300
+ #define BZ_X_BLKHDR_3 16
301
+ #define BZ_X_BLKHDR_4 17
302
+ #define BZ_X_BLKHDR_5 18
303
+ #define BZ_X_BLKHDR_6 19
304
+ #define BZ_X_BCRC_1 20
305
+ #define BZ_X_BCRC_2 21
306
+ #define BZ_X_BCRC_3 22
307
+ #define BZ_X_BCRC_4 23
308
+ #define BZ_X_RANDBIT 24
309
+ #define BZ_X_ORIGPTR_1 25
310
+ #define BZ_X_ORIGPTR_2 26
311
+ #define BZ_X_ORIGPTR_3 27
312
+ #define BZ_X_MAPPING_1 28
313
+ #define BZ_X_MAPPING_2 29
314
+ #define BZ_X_SELECTOR_1 30
315
+ #define BZ_X_SELECTOR_2 31
316
+ #define BZ_X_SELECTOR_3 32
317
+ #define BZ_X_CODING_1 33
318
+ #define BZ_X_CODING_2 34
319
+ #define BZ_X_CODING_3 35
320
+ #define BZ_X_MTF_1 36
321
+ #define BZ_X_MTF_2 37
322
+ #define BZ_X_MTF_3 38
323
+ #define BZ_X_MTF_4 39
324
+ #define BZ_X_MTF_5 40
325
+ #define BZ_X_MTF_6 41
326
+ #define BZ_X_ENDHDR_2 42
327
+ #define BZ_X_ENDHDR_3 43
328
+ #define BZ_X_ENDHDR_4 44
329
+ #define BZ_X_ENDHDR_5 45
330
+ #define BZ_X_ENDHDR_6 46
331
+ #define BZ_X_CCRC_1 47
332
+ #define BZ_X_CCRC_2 48
333
+ #define BZ_X_CCRC_3 49
334
+ #define BZ_X_CCRC_4 50
335
+
336
+
337
+
338
+ /*-- Constants for the fast MTF decoder. --*/
339
+
340
+ #define MTFA_SIZE 4096
341
+ #define MTFL_SIZE 16
342
+
343
+
344
+
345
+ /*-- Structure holding all the decompression-side stuff. --*/
346
+
347
+ typedef
348
+ struct {
349
+ /* pointer back to the struct bz_stream */
350
+ bz_stream* strm;
351
+
352
+ /* state indicator for this stream */
353
+ Int32 state;
354
+
355
+ /* for doing the final run-length decoding */
356
+ UChar state_out_ch;
357
+ Int32 state_out_len;
358
+ Bool blockRandomised;
359
+ BZ_RAND_DECLS;
360
+
361
+ /* the buffer for bit stream reading */
362
+ UInt32 bsBuff;
363
+ Int32 bsLive;
364
+
365
+ /* misc administratium */
366
+ Int32 blockSize100k;
367
+ Bool smallDecompress;
368
+ Int32 currBlockNo;
369
+ Int32 verbosity;
370
+
371
+ /* for undoing the Burrows-Wheeler transform */
372
+ Int32 origPtr;
373
+ UInt32 tPos;
374
+ Int32 k0;
375
+ Int32 unzftab[256];
376
+ Int32 nblock_used;
377
+ Int32 cftab[257];
378
+ Int32 cftabCopy[257];
379
+
380
+ /* for undoing the Burrows-Wheeler transform (FAST) */
381
+ UInt32 *tt;
382
+
383
+ /* for undoing the Burrows-Wheeler transform (SMALL) */
384
+ UInt16 *ll16;
385
+ UChar *ll4;
386
+
387
+ /* stored and calculated CRCs */
388
+ UInt32 storedBlockCRC;
389
+ UInt32 storedCombinedCRC;
390
+ UInt32 calculatedBlockCRC;
391
+ UInt32 calculatedCombinedCRC;
392
+
393
+ /* map of bytes used in block */
394
+ Int32 nInUse;
395
+ Bool inUse[256];
396
+ Bool inUse16[16];
397
+ UChar seqToUnseq[256];
398
+
399
+ /* for decoding the MTF values */
400
+ UChar mtfa [MTFA_SIZE];
401
+ Int32 mtfbase[256 / MTFL_SIZE];
402
+ UChar selector [BZ_MAX_SELECTORS];
403
+ UChar selectorMtf[BZ_MAX_SELECTORS];
404
+ UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
405
+
406
+ Int32 limit [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
407
+ Int32 base [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
408
+ Int32 perm [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
409
+ Int32 minLens[BZ_N_GROUPS];
410
+
411
+ /* save area for scalars in the main decompress code */
412
+ Int32 save_i;
413
+ Int32 save_j;
414
+ Int32 save_t;
415
+ Int32 save_alphaSize;
416
+ Int32 save_nGroups;
417
+ Int32 save_nSelectors;
418
+ Int32 save_EOB;
419
+ Int32 save_groupNo;
420
+ Int32 save_groupPos;
421
+ Int32 save_nextSym;
422
+ Int32 save_nblockMAX;
423
+ Int32 save_nblock;
424
+ Int32 save_es;
425
+ Int32 save_N;
426
+ Int32 save_curr;
427
+ Int32 save_zt;
428
+ Int32 save_zn;
429
+ Int32 save_zvec;
430
+ Int32 save_zj;
431
+ Int32 save_gSel;
432
+ Int32 save_gMinlen;
433
+ Int32* save_gLimit;
434
+ Int32* save_gBase;
435
+ Int32* save_gPerm;
436
+
437
+ }
438
+ DState;
439
+
440
+
441
+
442
+ /*-- Macros for decompression. --*/
443
+
444
+ #define BZ_GET_FAST(cccc) \
445
+ /* c_tPos is unsigned, hence test < 0 is pointless. */ \
446
+ if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \
447
+ s->tPos = s->tt[s->tPos]; \
448
+ cccc = (UChar)(s->tPos & 0xff); \
449
+ s->tPos >>= 8;
450
+
451
+ #define BZ_GET_FAST_C(cccc) \
452
+ /* c_tPos is unsigned, hence test < 0 is pointless. */ \
453
+ if (c_tPos >= (UInt32)100000 * (UInt32)ro_blockSize100k) return True; \
454
+ c_tPos = c_tt[c_tPos]; \
455
+ cccc = (UChar)(c_tPos & 0xff); \
456
+ c_tPos >>= 8;
457
+
458
+ #define SET_LL4(i,n) \
459
+ { if (((i) & 0x1) == 0) \
460
+ s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0xf0) | (n); else \
461
+ s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0x0f) | ((n) << 4); \
462
+ }
463
+
464
+ #define GET_LL4(i) \
465
+ ((((UInt32)(s->ll4[(i) >> 1])) >> (((i) << 2) & 0x4)) & 0xF)
466
+
467
+ #define SET_LL(i,n) \
468
+ { s->ll16[i] = (UInt16)(n & 0x0000ffff); \
469
+ SET_LL4(i, n >> 16); \
470
+ }
471
+
472
+ #define GET_LL(i) \
473
+ (((UInt32)s->ll16[i]) | (GET_LL4(i) << 16))
474
+
475
+ #define BZ_GET_SMALL(cccc) \
476
+ /* c_tPos is unsigned, hence test < 0 is pointless. */ \
477
+ if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \
478
+ cccc = BZ2_indexIntoF ( s->tPos, s->cftab ); \
479
+ s->tPos = GET_LL(s->tPos);
480
+
481
+
482
+ /*-- externs for decompression. --*/
483
+
484
+ extern Int32
485
+ BZ2_indexIntoF ( Int32, Int32* );
486
+
487
+ extern Int32
488
+ BZ2_decompress ( DState* );
489
+
490
+ extern void
491
+ BZ2_hbCreateDecodeTables ( Int32*, Int32*, Int32*, UChar*,
492
+ Int32, Int32, Int32 );
493
+
494
+
495
+ #endif
496
+
497
+
498
+ /*-- BZ_NO_STDIO seems to make NULL disappear on some platforms. --*/
499
+
500
+ #ifdef BZ_NO_STDIO
501
+ #ifndef NULL
502
+ #define NULL 0
503
+ #endif
504
+ #endif
505
+
506
+
507
+ /*-------------------------------------------------------------*/
508
+ /*--- end bzlib_private.h ---*/
509
+ /*-------------------------------------------------------------*/
@@ -0,0 +1,84 @@
1
+
2
+ /*-------------------------------------------------------------*/
3
+ /*--- Table for randomising repetitive blocks ---*/
4
+ /*--- randtable.c ---*/
5
+ /*-------------------------------------------------------------*/
6
+
7
+ /* ------------------------------------------------------------------
8
+ This file is part of bzip2/libbzip2, a program and library for
9
+ lossless, block-sorting data compression.
10
+
11
+ bzip2/libbzip2 version 1.0.8 of 13 July 2019
12
+ Copyright (C) 1996-2019 Julian Seward <jseward@acm.org>
13
+
14
+ Please read the WARNING, DISCLAIMER and PATENTS sections in the
15
+ README file.
16
+
17
+ This program is released under the terms of the license contained
18
+ in the file LICENSE.
19
+ ------------------------------------------------------------------ */
20
+
21
+
22
+ #include "bzlib_private.h"
23
+
24
+
25
+ /*---------------------------------------------*/
26
+ Int32 BZ2_rNums[512] = {
27
+ 619, 720, 127, 481, 931, 816, 813, 233, 566, 247,
28
+ 985, 724, 205, 454, 863, 491, 741, 242, 949, 214,
29
+ 733, 859, 335, 708, 621, 574, 73, 654, 730, 472,
30
+ 419, 436, 278, 496, 867, 210, 399, 680, 480, 51,
31
+ 878, 465, 811, 169, 869, 675, 611, 697, 867, 561,
32
+ 862, 687, 507, 283, 482, 129, 807, 591, 733, 623,
33
+ 150, 238, 59, 379, 684, 877, 625, 169, 643, 105,
34
+ 170, 607, 520, 932, 727, 476, 693, 425, 174, 647,
35
+ 73, 122, 335, 530, 442, 853, 695, 249, 445, 515,
36
+ 909, 545, 703, 919, 874, 474, 882, 500, 594, 612,
37
+ 641, 801, 220, 162, 819, 984, 589, 513, 495, 799,
38
+ 161, 604, 958, 533, 221, 400, 386, 867, 600, 782,
39
+ 382, 596, 414, 171, 516, 375, 682, 485, 911, 276,
40
+ 98, 553, 163, 354, 666, 933, 424, 341, 533, 870,
41
+ 227, 730, 475, 186, 263, 647, 537, 686, 600, 224,
42
+ 469, 68, 770, 919, 190, 373, 294, 822, 808, 206,
43
+ 184, 943, 795, 384, 383, 461, 404, 758, 839, 887,
44
+ 715, 67, 618, 276, 204, 918, 873, 777, 604, 560,
45
+ 951, 160, 578, 722, 79, 804, 96, 409, 713, 940,
46
+ 652, 934, 970, 447, 318, 353, 859, 672, 112, 785,
47
+ 645, 863, 803, 350, 139, 93, 354, 99, 820, 908,
48
+ 609, 772, 154, 274, 580, 184, 79, 626, 630, 742,
49
+ 653, 282, 762, 623, 680, 81, 927, 626, 789, 125,
50
+ 411, 521, 938, 300, 821, 78, 343, 175, 128, 250,
51
+ 170, 774, 972, 275, 999, 639, 495, 78, 352, 126,
52
+ 857, 956, 358, 619, 580, 124, 737, 594, 701, 612,
53
+ 669, 112, 134, 694, 363, 992, 809, 743, 168, 974,
54
+ 944, 375, 748, 52, 600, 747, 642, 182, 862, 81,
55
+ 344, 805, 988, 739, 511, 655, 814, 334, 249, 515,
56
+ 897, 955, 664, 981, 649, 113, 974, 459, 893, 228,
57
+ 433, 837, 553, 268, 926, 240, 102, 654, 459, 51,
58
+ 686, 754, 806, 760, 493, 403, 415, 394, 687, 700,
59
+ 946, 670, 656, 610, 738, 392, 760, 799, 887, 653,
60
+ 978, 321, 576, 617, 626, 502, 894, 679, 243, 440,
61
+ 680, 879, 194, 572, 640, 724, 926, 56, 204, 700,
62
+ 707, 151, 457, 449, 797, 195, 791, 558, 945, 679,
63
+ 297, 59, 87, 824, 713, 663, 412, 693, 342, 606,
64
+ 134, 108, 571, 364, 631, 212, 174, 643, 304, 329,
65
+ 343, 97, 430, 751, 497, 314, 983, 374, 822, 928,
66
+ 140, 206, 73, 263, 980, 736, 876, 478, 430, 305,
67
+ 170, 514, 364, 692, 829, 82, 855, 953, 676, 246,
68
+ 369, 970, 294, 750, 807, 827, 150, 790, 288, 923,
69
+ 804, 378, 215, 828, 592, 281, 565, 555, 710, 82,
70
+ 896, 831, 547, 261, 524, 462, 293, 465, 502, 56,
71
+ 661, 821, 976, 991, 658, 869, 905, 758, 745, 193,
72
+ 768, 550, 608, 933, 378, 286, 215, 979, 792, 961,
73
+ 61, 688, 793, 644, 986, 403, 106, 366, 905, 644,
74
+ 372, 567, 466, 434, 645, 210, 389, 550, 919, 135,
75
+ 780, 773, 635, 389, 707, 100, 626, 958, 165, 504,
76
+ 920, 176, 193, 713, 857, 265, 203, 50, 668, 108,
77
+ 645, 990, 626, 197, 510, 357, 358, 850, 858, 364,
78
+ 936, 638
79
+ };
80
+
81
+
82
+ /*-------------------------------------------------------------*/
83
+ /*--- end randtable.c ---*/
84
+ /*-------------------------------------------------------------*/
@@ -1,53 +1,53 @@
1
- /*
2
- Copyright: Boaz Segev, 2018-2019
3
- License: MIT
4
- */
5
- #ifndef H_FIO_TMPFILE_H
6
- /** a simple helper to create temporary files and file names */
7
- #define H_FIO_TMPFILE_H
8
-
9
- #ifndef _GNU_SOURCE
10
- #define _GNU_SOURCE
11
- #endif
12
-
13
- #include <stdio.h>
14
- #include <stdlib.h>
15
-
16
- #include <fcntl.h>
17
- #include <sys/stat.h>
18
- #include <sys/types.h>
19
- #include <unistd.h>
20
-
21
- #ifdef __MINGW32__
22
- #include <fileapi.h>
23
- #endif
24
-
25
- static inline int fio_tmpfile(void) {
26
- // create a temporary file to contain the data.
27
- int fd = 0;
28
- #ifdef __MINGW32__
29
- char name_template[] = "fio";
30
- TCHAR temp_path[(MAX_PATH-14)];
31
- TCHAR temp_filename[MAX_PATH];
32
- GetTempPath(MAX_PATH - 14, temp_path);
33
- GetTempFileNameA(temp_path, name_template, 0, temp_filename);
34
- fd = _open(temp_filename, _O_CREAT | _O_RDWR);
35
- _chmod(temp_filename, _S_IREAD | _S_IWRITE);
36
- #elif defined(P_tmpdir)
37
- if (P_tmpdir[sizeof(P_tmpdir) - 1] == '/') {
38
- char name_template[] = P_tmpdir "facil_io_tmpfile_XXXXXXXX";
39
- fd = mkstemp(name_template);
40
- } else {
41
- char name_template[] = P_tmpdir "/facil_io_tmpfile_XXXXXXXX";
42
- fd = mkstemp(name_template);
43
- }
44
- #else
45
- char name_template[] = "/tmp/facil_io_tmpfile_XXXXXXXX";
46
- fd = mkstemp(name_template);
47
- #endif
48
- return fd;
49
- }
50
-
51
- #endif
52
-
53
-
1
+ /*
2
+ Copyright: Boaz Segev, 2018-2019
3
+ License: MIT
4
+ */
5
+ #ifndef H_FIO_TMPFILE_H
6
+ /** a simple helper to create temporary files and file names */
7
+ #define H_FIO_TMPFILE_H
8
+
9
+ #ifndef _GNU_SOURCE
10
+ #define _GNU_SOURCE
11
+ #endif
12
+
13
+ #include <stdio.h>
14
+ #include <stdlib.h>
15
+
16
+ #include <fcntl.h>
17
+ #include <sys/stat.h>
18
+ #include <sys/types.h>
19
+ #include <unistd.h>
20
+
21
+ #ifdef __MINGW32__
22
+ #include <fileapi.h>
23
+ #endif
24
+
25
+ static inline int fio_tmpfile(void) {
26
+ // create a temporary file to contain the data.
27
+ int fd = 0;
28
+ #ifdef __MINGW32__
29
+ char name_template[] = "fio";
30
+ TCHAR temp_path[(MAX_PATH-14)];
31
+ TCHAR temp_filename[MAX_PATH];
32
+ GetTempPath(MAX_PATH - 14, temp_path);
33
+ GetTempFileNameA(temp_path, name_template, 0, temp_filename);
34
+ fd = _open(temp_filename, _O_CREAT | _O_RDWR);
35
+ _chmod(temp_filename, _S_IREAD | _S_IWRITE);
36
+ #elif defined(P_tmpdir)
37
+ if (P_tmpdir[sizeof(P_tmpdir) - 1] == '/') {
38
+ char name_template[] = P_tmpdir "facil_io_tmpfile_XXXXXXXX";
39
+ fd = mkstemp(name_template);
40
+ } else {
41
+ char name_template[] = P_tmpdir "/facil_io_tmpfile_XXXXXXXX";
42
+ fd = mkstemp(name_template);
43
+ }
44
+ #else
45
+ char name_template[] = "/tmp/facil_io_tmpfile_XXXXXXXX";
46
+ fd = mkstemp(name_template);
47
+ #endif
48
+ return fd;
49
+ }
50
+
51
+ #endif
52
+
53
+