isomorfeus-ferret 0.12.7 → 0.13.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (166) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +101 -19
  3. data/README.md +85 -13
  4. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +22 -30
  5. data/ext/isomorfeus_ferret_ext/bm_hash.c +6 -12
  6. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +3 -6
  7. data/ext/isomorfeus_ferret_ext/bm_store.c +11 -22
  8. data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.c +1 -1
  9. data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +1 -1
  10. data/ext/isomorfeus_ferret_ext/bzlib.c +1572 -0
  11. data/ext/isomorfeus_ferret_ext/bzlib.h +282 -0
  12. data/ext/isomorfeus_ferret_ext/bzlib_blocksort.c +1094 -0
  13. data/ext/isomorfeus_ferret_ext/bzlib_compress.c +672 -0
  14. data/ext/isomorfeus_ferret_ext/bzlib_crctable.c +104 -0
  15. data/ext/isomorfeus_ferret_ext/bzlib_decompress.c +652 -0
  16. data/ext/isomorfeus_ferret_ext/bzlib_huffman.c +205 -0
  17. data/ext/isomorfeus_ferret_ext/bzlib_private.h +509 -0
  18. data/ext/isomorfeus_ferret_ext/bzlib_randtable.c +84 -0
  19. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -53
  20. data/ext/isomorfeus_ferret_ext/frb_analysis.c +785 -1192
  21. data/ext/isomorfeus_ferret_ext/frb_index.c +497 -495
  22. data/ext/isomorfeus_ferret_ext/frb_qparser.c +48 -60
  23. data/ext/isomorfeus_ferret_ext/frb_search.c +1520 -1002
  24. data/ext/isomorfeus_ferret_ext/frb_store.c +96 -96
  25. data/ext/isomorfeus_ferret_ext/frb_threading.h +0 -1
  26. data/ext/isomorfeus_ferret_ext/frb_utils.c +147 -196
  27. data/ext/isomorfeus_ferret_ext/frt_analysis.c +695 -1090
  28. data/ext/isomorfeus_ferret_ext/frt_analysis.h +174 -170
  29. data/ext/isomorfeus_ferret_ext/frt_array.c +2 -4
  30. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +9 -16
  31. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +32 -81
  32. data/ext/isomorfeus_ferret_ext/frt_document.c +15 -20
  33. data/ext/isomorfeus_ferret_ext/frt_document.h +10 -10
  34. data/ext/isomorfeus_ferret_ext/frt_except.c +5 -12
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +3 -3
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +6 -7
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +35 -46
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +1 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +91 -200
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +7 -18
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +1 -2
  42. data/ext/isomorfeus_ferret_ext/frt_ind.c +32 -35
  43. data/ext/isomorfeus_ferret_ext/frt_ind.h +9 -9
  44. data/ext/isomorfeus_ferret_ext/frt_index.c +603 -410
  45. data/ext/isomorfeus_ferret_ext/frt_index.h +272 -291
  46. data/ext/isomorfeus_ferret_ext/frt_lang.c +0 -2
  47. data/ext/isomorfeus_ferret_ext/frt_mempool.c +1 -2
  48. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +4 -7
  49. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +68 -91
  50. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +35 -38
  51. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +53 -72
  52. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +25 -32
  53. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +21 -23
  54. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +66 -103
  55. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +207 -195
  56. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +20 -16
  57. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +17 -14
  58. data/ext/isomorfeus_ferret_ext/frt_q_range.c +102 -131
  59. data/ext/isomorfeus_ferret_ext/frt_q_span.c +179 -178
  60. data/ext/isomorfeus_ferret_ext/frt_q_term.c +47 -60
  61. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +18 -16
  62. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +45 -84
  63. data/ext/isomorfeus_ferret_ext/frt_search.c +105 -146
  64. data/ext/isomorfeus_ferret_ext/frt_search.h +331 -320
  65. data/ext/isomorfeus_ferret_ext/frt_similarity.c +5 -13
  66. data/ext/isomorfeus_ferret_ext/frt_similarity.h +7 -12
  67. data/ext/isomorfeus_ferret_ext/frt_sort.c +105 -149
  68. data/ext/isomorfeus_ferret_ext/frt_store.c +13 -7
  69. data/ext/isomorfeus_ferret_ext/frt_store.h +10 -2
  70. data/ext/isomorfeus_ferret_ext/frt_threading.h +0 -1
  71. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +22 -112
  72. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +2 -32
  73. data/ext/isomorfeus_ferret_ext/lz4.c +2495 -0
  74. data/ext/isomorfeus_ferret_ext/lz4.h +774 -0
  75. data/ext/isomorfeus_ferret_ext/lz4frame.c +1899 -0
  76. data/ext/isomorfeus_ferret_ext/lz4frame.h +623 -0
  77. data/ext/isomorfeus_ferret_ext/lz4hc.c +1615 -0
  78. data/ext/isomorfeus_ferret_ext/lz4hc.h +413 -0
  79. data/ext/isomorfeus_ferret_ext/lz4xxhash.c +1030 -0
  80. data/ext/isomorfeus_ferret_ext/lz4xxhash.h +328 -0
  81. data/ext/isomorfeus_ferret_ext/stem_modules.h +0 -86
  82. data/ext/isomorfeus_ferret_ext/test.c +0 -17
  83. data/ext/isomorfeus_ferret_ext/test_1710.c +11 -12
  84. data/ext/isomorfeus_ferret_ext/test_analysis.c +590 -583
  85. data/ext/isomorfeus_ferret_ext/test_compound_io.c +1 -1
  86. data/ext/isomorfeus_ferret_ext/test_document.c +19 -15
  87. data/ext/isomorfeus_ferret_ext/test_except.c +1 -2
  88. data/ext/isomorfeus_ferret_ext/test_fields.c +59 -60
  89. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +10 -27
  90. data/ext/isomorfeus_ferret_ext/test_filter.c +11 -8
  91. data/ext/isomorfeus_ferret_ext/test_global.c +0 -46
  92. data/ext/isomorfeus_ferret_ext/test_hash.c +2 -2
  93. data/ext/isomorfeus_ferret_ext/test_hashset.c +1 -1
  94. data/ext/isomorfeus_ferret_ext/test_highlighter.c +15 -11
  95. data/ext/isomorfeus_ferret_ext/test_index.c +372 -365
  96. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +5 -3
  97. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +5 -3
  98. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +13 -10
  99. data/ext/isomorfeus_ferret_ext/test_q_parser.c +45 -7
  100. data/ext/isomorfeus_ferret_ext/test_q_span.c +15 -12
  101. data/ext/isomorfeus_ferret_ext/test_ram_store.c +3 -3
  102. data/ext/isomorfeus_ferret_ext/test_search.c +60 -64
  103. data/ext/isomorfeus_ferret_ext/test_segments.c +5 -4
  104. data/ext/isomorfeus_ferret_ext/test_sort.c +17 -14
  105. data/ext/isomorfeus_ferret_ext/test_store.c +2 -0
  106. data/ext/isomorfeus_ferret_ext/test_term.c +3 -1
  107. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -10
  108. data/ext/isomorfeus_ferret_ext/test_test.c +1 -2
  109. data/ext/isomorfeus_ferret_ext/test_threading.c +9 -10
  110. data/ext/isomorfeus_ferret_ext/testhelper.c +1 -2
  111. data/lib/isomorfeus/ferret/version.rb +1 -1
  112. metadata +27 -57
  113. data/ext/isomorfeus_ferret_ext/email.rl +0 -21
  114. data/ext/isomorfeus_ferret_ext/frt_scanner.c +0 -900
  115. data/ext/isomorfeus_ferret_ext/frt_scanner.h +0 -28
  116. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +0 -6706
  117. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +0 -4420
  118. data/ext/isomorfeus_ferret_ext/scanner.h +0 -28
  119. data/ext/isomorfeus_ferret_ext/scanner.in +0 -43
  120. data/ext/isomorfeus_ferret_ext/scanner.rl +0 -84
  121. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +0 -200
  122. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +0 -85
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.c +0 -1167
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.h +0 -6
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.c +0 -1433
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.h +0 -6
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +0 -301
  128. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +0 -6
  129. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +0 -590
  130. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +0 -6
  131. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +0 -1049
  132. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +0 -6
  133. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +0 -705
  134. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +0 -6
  135. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +0 -1239
  136. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +0 -6
  137. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +0 -477
  138. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +0 -6
  139. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +0 -1217
  140. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +0 -7
  141. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.c +0 -394
  142. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.h +0 -6
  143. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.c +0 -457
  144. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.h +0 -6
  145. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +0 -1009
  146. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +0 -6
  147. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +0 -259
  148. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +0 -6
  149. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +0 -704
  150. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +0 -6
  151. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +0 -948
  152. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +0 -6
  153. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +0 -1028
  154. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +0 -6
  155. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +0 -275
  156. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +0 -6
  157. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.c +0 -849
  158. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.h +0 -6
  159. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +0 -952
  160. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +0 -6
  161. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +0 -669
  162. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +0 -6
  163. data/ext/isomorfeus_ferret_ext/stem_modules.txt +0 -63
  164. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +0 -1854
  165. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +0 -1999
  166. data/ext/isomorfeus_ferret_ext/url.rl +0 -27
@@ -0,0 +1,205 @@
1
+
2
+ /*-------------------------------------------------------------*/
3
+ /*--- Huffman coding low-level stuff ---*/
4
+ /*--- huffman.c ---*/
5
+ /*-------------------------------------------------------------*/
6
+
7
+ /* ------------------------------------------------------------------
8
+ This file is part of bzip2/libbzip2, a program and library for
9
+ lossless, block-sorting data compression.
10
+
11
+ bzip2/libbzip2 version 1.0.8 of 13 July 2019
12
+ Copyright (C) 1996-2019 Julian Seward <jseward@acm.org>
13
+
14
+ Please read the WARNING, DISCLAIMER and PATENTS sections in the
15
+ README file.
16
+
17
+ This program is released under the terms of the license contained
18
+ in the file LICENSE.
19
+ ------------------------------------------------------------------ */
20
+
21
+
22
+ #include "bzlib_private.h"
23
+
24
+ /*---------------------------------------------------*/
25
+ #define WEIGHTOF(zz0) ((zz0) & 0xffffff00)
26
+ #define DEPTHOF(zz1) ((zz1) & 0x000000ff)
27
+ #define MYMAX(zz2,zz3) ((zz2) > (zz3) ? (zz2) : (zz3))
28
+
29
+ #define ADDWEIGHTS(zw1,zw2) \
30
+ (WEIGHTOF(zw1)+WEIGHTOF(zw2)) | \
31
+ (1 + MYMAX(DEPTHOF(zw1),DEPTHOF(zw2)))
32
+
33
+ #define UPHEAP(z) \
34
+ { \
35
+ Int32 zz, tmp; \
36
+ zz = z; tmp = heap[zz]; \
37
+ while (weight[tmp] < weight[heap[zz >> 1]]) { \
38
+ heap[zz] = heap[zz >> 1]; \
39
+ zz >>= 1; \
40
+ } \
41
+ heap[zz] = tmp; \
42
+ }
43
+
44
+ #define DOWNHEAP(z) \
45
+ { \
46
+ Int32 zz, yy, tmp; \
47
+ zz = z; tmp = heap[zz]; \
48
+ while (True) { \
49
+ yy = zz << 1; \
50
+ if (yy > nHeap) break; \
51
+ if (yy < nHeap && \
52
+ weight[heap[yy+1]] < weight[heap[yy]]) \
53
+ yy++; \
54
+ if (weight[tmp] < weight[heap[yy]]) break; \
55
+ heap[zz] = heap[yy]; \
56
+ zz = yy; \
57
+ } \
58
+ heap[zz] = tmp; \
59
+ }
60
+
61
+
62
+ /*---------------------------------------------------*/
63
+ void BZ2_hbMakeCodeLengths ( UChar *len,
64
+ Int32 *freq,
65
+ Int32 alphaSize,
66
+ Int32 maxLen )
67
+ {
68
+ /*--
69
+ Nodes and heap entries run from 1. Entry 0
70
+ for both the heap and nodes is a sentinel.
71
+ --*/
72
+ Int32 nNodes, nHeap, n1, n2, i, j, k;
73
+ Bool tooLong;
74
+
75
+ Int32 heap [ BZ_MAX_ALPHA_SIZE + 2 ];
76
+ Int32 weight [ BZ_MAX_ALPHA_SIZE * 2 ];
77
+ Int32 parent [ BZ_MAX_ALPHA_SIZE * 2 ];
78
+
79
+ for (i = 0; i < alphaSize; i++)
80
+ weight[i+1] = (freq[i] == 0 ? 1 : freq[i]) << 8;
81
+
82
+ while (True) {
83
+
84
+ nNodes = alphaSize;
85
+ nHeap = 0;
86
+
87
+ heap[0] = 0;
88
+ weight[0] = 0;
89
+ parent[0] = -2;
90
+
91
+ for (i = 1; i <= alphaSize; i++) {
92
+ parent[i] = -1;
93
+ nHeap++;
94
+ heap[nHeap] = i;
95
+ UPHEAP(nHeap);
96
+ }
97
+
98
+ AssertH( nHeap < (BZ_MAX_ALPHA_SIZE+2), 2001 );
99
+
100
+ while (nHeap > 1) {
101
+ n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1);
102
+ n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1);
103
+ nNodes++;
104
+ parent[n1] = parent[n2] = nNodes;
105
+ weight[nNodes] = ADDWEIGHTS(weight[n1], weight[n2]);
106
+ parent[nNodes] = -1;
107
+ nHeap++;
108
+ heap[nHeap] = nNodes;
109
+ UPHEAP(nHeap);
110
+ }
111
+
112
+ AssertH( nNodes < (BZ_MAX_ALPHA_SIZE * 2), 2002 );
113
+
114
+ tooLong = False;
115
+ for (i = 1; i <= alphaSize; i++) {
116
+ j = 0;
117
+ k = i;
118
+ while (parent[k] >= 0) { k = parent[k]; j++; }
119
+ len[i-1] = j;
120
+ if (j > maxLen) tooLong = True;
121
+ }
122
+
123
+ if (! tooLong) break;
124
+
125
+ /* 17 Oct 04: keep-going condition for the following loop used
126
+ to be 'i < alphaSize', which missed the last element,
127
+ theoretically leading to the possibility of the compressor
128
+ looping. However, this count-scaling step is only needed if
129
+ one of the generated Huffman code words is longer than
130
+ maxLen, which up to and including version 1.0.2 was 20 bits,
131
+ which is extremely unlikely. In version 1.0.3 maxLen was
132
+ changed to 17 bits, which has minimal effect on compression
133
+ ratio, but does mean this scaling step is used from time to
134
+ time, enough to verify that it works.
135
+
136
+ This means that bzip2-1.0.3 and later will only produce
137
+ Huffman codes with a maximum length of 17 bits. However, in
138
+ order to preserve backwards compatibility with bitstreams
139
+ produced by versions pre-1.0.3, the decompressor must still
140
+ handle lengths of up to 20. */
141
+
142
+ for (i = 1; i <= alphaSize; i++) {
143
+ j = weight[i] >> 8;
144
+ j = 1 + (j / 2);
145
+ weight[i] = j << 8;
146
+ }
147
+ }
148
+ }
149
+
150
+
151
+ /*---------------------------------------------------*/
152
+ void BZ2_hbAssignCodes ( Int32 *code,
153
+ UChar *length,
154
+ Int32 minLen,
155
+ Int32 maxLen,
156
+ Int32 alphaSize )
157
+ {
158
+ Int32 n, vec, i;
159
+
160
+ vec = 0;
161
+ for (n = minLen; n <= maxLen; n++) {
162
+ for (i = 0; i < alphaSize; i++)
163
+ if (length[i] == n) { code[i] = vec; vec++; };
164
+ vec <<= 1;
165
+ }
166
+ }
167
+
168
+
169
+ /*---------------------------------------------------*/
170
+ void BZ2_hbCreateDecodeTables ( Int32 *limit,
171
+ Int32 *base,
172
+ Int32 *perm,
173
+ UChar *length,
174
+ Int32 minLen,
175
+ Int32 maxLen,
176
+ Int32 alphaSize )
177
+ {
178
+ Int32 pp, i, j, vec;
179
+
180
+ pp = 0;
181
+ for (i = minLen; i <= maxLen; i++)
182
+ for (j = 0; j < alphaSize; j++)
183
+ if (length[j] == i) { perm[pp] = j; pp++; };
184
+
185
+ for (i = 0; i < BZ_MAX_CODE_LEN; i++) base[i] = 0;
186
+ for (i = 0; i < alphaSize; i++) base[length[i]+1]++;
187
+
188
+ for (i = 1; i < BZ_MAX_CODE_LEN; i++) base[i] += base[i-1];
189
+
190
+ for (i = 0; i < BZ_MAX_CODE_LEN; i++) limit[i] = 0;
191
+ vec = 0;
192
+
193
+ for (i = minLen; i <= maxLen; i++) {
194
+ vec += (base[i+1] - base[i]);
195
+ limit[i] = vec-1;
196
+ vec <<= 1;
197
+ }
198
+ for (i = minLen + 1; i <= maxLen; i++)
199
+ base[i] = ((limit[i-1] + 1) << 1) - base[i];
200
+ }
201
+
202
+
203
+ /*-------------------------------------------------------------*/
204
+ /*--- end huffman.c ---*/
205
+ /*-------------------------------------------------------------*/
@@ -0,0 +1,509 @@
1
+
2
+ /*-------------------------------------------------------------*/
3
+ /*--- Private header file for the library. ---*/
4
+ /*--- bzlib_private.h ---*/
5
+ /*-------------------------------------------------------------*/
6
+
7
+ /* ------------------------------------------------------------------
8
+ This file is part of bzip2/libbzip2, a program and library for
9
+ lossless, block-sorting data compression.
10
+
11
+ bzip2/libbzip2 version 1.0.8 of 13 July 2019
12
+ Copyright (C) 1996-2019 Julian Seward <jseward@acm.org>
13
+
14
+ Please read the WARNING, DISCLAIMER and PATENTS sections in the
15
+ README file.
16
+
17
+ This program is released under the terms of the license contained
18
+ in the file LICENSE.
19
+ ------------------------------------------------------------------ */
20
+
21
+
22
+ #ifndef _BZLIB_PRIVATE_H
23
+ #define _BZLIB_PRIVATE_H
24
+
25
+ #include <stdlib.h>
26
+
27
+ #ifndef BZ_NO_STDIO
28
+ #include <stdio.h>
29
+ #include <ctype.h>
30
+ #include <string.h>
31
+ #endif
32
+
33
+ #include "bzlib.h"
34
+
35
+
36
+
37
+ /*-- General stuff. --*/
38
+
39
+ #define BZ_VERSION "1.0.8, 13-Jul-2019"
40
+
41
+ typedef char Char;
42
+ typedef unsigned char Bool;
43
+ typedef unsigned char UChar;
44
+ typedef int Int32;
45
+ typedef unsigned int UInt32;
46
+ typedef short Int16;
47
+ typedef unsigned short UInt16;
48
+
49
+ #define True ((Bool)1)
50
+ #define False ((Bool)0)
51
+
52
+ #ifndef __GNUC__
53
+ #define __inline__ /* */
54
+ #endif
55
+
56
+ #ifndef BZ_NO_STDIO
57
+
58
+ extern void BZ2_bz__AssertH__fail ( int errcode );
59
+ #define AssertH(cond,errcode) \
60
+ { if (!(cond)) BZ2_bz__AssertH__fail ( errcode ); }
61
+
62
+ #if BZ_DEBUG
63
+ #define AssertD(cond,msg) \
64
+ { if (!(cond)) { \
65
+ fprintf ( stderr, \
66
+ "\n\nlibbzip2(debug build): internal error\n\t%s\n", msg );\
67
+ exit(1); \
68
+ }}
69
+ #else
70
+ #define AssertD(cond,msg) /* */
71
+ #endif
72
+
73
+ #define VPrintf0(zf) \
74
+ fprintf(stderr,zf)
75
+ #define VPrintf1(zf,za1) \
76
+ fprintf(stderr,zf,za1)
77
+ #define VPrintf2(zf,za1,za2) \
78
+ fprintf(stderr,zf,za1,za2)
79
+ #define VPrintf3(zf,za1,za2,za3) \
80
+ fprintf(stderr,zf,za1,za2,za3)
81
+ #define VPrintf4(zf,za1,za2,za3,za4) \
82
+ fprintf(stderr,zf,za1,za2,za3,za4)
83
+ #define VPrintf5(zf,za1,za2,za3,za4,za5) \
84
+ fprintf(stderr,zf,za1,za2,za3,za4,za5)
85
+
86
+ #else
87
+
88
+ extern void bz_internal_error ( int errcode );
89
+ #define AssertH(cond,errcode) \
90
+ { if (!(cond)) bz_internal_error ( errcode ); }
91
+ #define AssertD(cond,msg) do { } while (0)
92
+ #define VPrintf0(zf) do { } while (0)
93
+ #define VPrintf1(zf,za1) do { } while (0)
94
+ #define VPrintf2(zf,za1,za2) do { } while (0)
95
+ #define VPrintf3(zf,za1,za2,za3) do { } while (0)
96
+ #define VPrintf4(zf,za1,za2,za3,za4) do { } while (0)
97
+ #define VPrintf5(zf,za1,za2,za3,za4,za5) do { } while (0)
98
+
99
+ #endif
100
+
101
+
102
+ #define BZALLOC(nnn) (strm->bzalloc)(strm->opaque,(nnn),1)
103
+ #define BZFREE(ppp) (strm->bzfree)(strm->opaque,(ppp))
104
+
105
+
106
+ /*-- Header bytes. --*/
107
+
108
+ #define BZ_HDR_B 0x42 /* 'B' */
109
+ #define BZ_HDR_Z 0x5a /* 'Z' */
110
+ #define BZ_HDR_h 0x68 /* 'h' */
111
+ #define BZ_HDR_0 0x30 /* '0' */
112
+
113
+ /*-- Constants for the back end. --*/
114
+
115
+ #define BZ_MAX_ALPHA_SIZE 258
116
+ #define BZ_MAX_CODE_LEN 23
117
+
118
+ #define BZ_RUNA 0
119
+ #define BZ_RUNB 1
120
+
121
+ #define BZ_N_GROUPS 6
122
+ #define BZ_G_SIZE 50
123
+ #define BZ_N_ITERS 4
124
+
125
+ #define BZ_MAX_SELECTORS (2 + (900000 / BZ_G_SIZE))
126
+
127
+
128
+
129
+ /*-- Stuff for randomising repetitive blocks. --*/
130
+
131
+ extern Int32 BZ2_rNums[512];
132
+
133
+ #define BZ_RAND_DECLS \
134
+ Int32 rNToGo; \
135
+ Int32 rTPos \
136
+
137
+ #define BZ_RAND_INIT_MASK \
138
+ s->rNToGo = 0; \
139
+ s->rTPos = 0 \
140
+
141
+ #define BZ_RAND_MASK ((s->rNToGo == 1) ? 1 : 0)
142
+
143
+ #define BZ_RAND_UPD_MASK \
144
+ if (s->rNToGo == 0) { \
145
+ s->rNToGo = BZ2_rNums[s->rTPos]; \
146
+ s->rTPos++; \
147
+ if (s->rTPos == 512) s->rTPos = 0; \
148
+ } \
149
+ s->rNToGo--;
150
+
151
+
152
+
153
+ /*-- Stuff for doing CRCs. --*/
154
+
155
+ extern UInt32 BZ2_crc32Table[256];
156
+
157
+ #define BZ_INITIALISE_CRC(crcVar) \
158
+ { \
159
+ crcVar = 0xffffffffL; \
160
+ }
161
+
162
+ #define BZ_FINALISE_CRC(crcVar) \
163
+ { \
164
+ crcVar = ~(crcVar); \
165
+ }
166
+
167
+ #define BZ_UPDATE_CRC(crcVar,cha) \
168
+ { \
169
+ crcVar = (crcVar << 8) ^ \
170
+ BZ2_crc32Table[(crcVar >> 24) ^ \
171
+ ((UChar)cha)]; \
172
+ }
173
+
174
+
175
+
176
+ /*-- States and modes for compression. --*/
177
+
178
+ #define BZ_M_IDLE 1
179
+ #define BZ_M_RUNNING 2
180
+ #define BZ_M_FLUSHING 3
181
+ #define BZ_M_FINISHING 4
182
+
183
+ #define BZ_S_OUTPUT 1
184
+ #define BZ_S_INPUT 2
185
+
186
+ #define BZ_N_RADIX 2
187
+ #define BZ_N_QSORT 12
188
+ #define BZ_N_SHELL 18
189
+ #define BZ_N_OVERSHOOT (BZ_N_RADIX + BZ_N_QSORT + BZ_N_SHELL + 2)
190
+
191
+
192
+
193
+
194
+ /*-- Structure holding all the compression-side stuff. --*/
195
+
196
+ typedef
197
+ struct {
198
+ /* pointer back to the struct bz_stream */
199
+ bz_stream* strm;
200
+
201
+ /* mode this stream is in, and whether inputting */
202
+ /* or outputting data */
203
+ Int32 mode;
204
+ Int32 state;
205
+
206
+ /* remembers avail_in when flush/finish requested */
207
+ UInt32 avail_in_expect;
208
+
209
+ /* for doing the block sorting */
210
+ UInt32* arr1;
211
+ UInt32* arr2;
212
+ UInt32* ftab;
213
+ Int32 origPtr;
214
+
215
+ /* aliases for arr1 and arr2 */
216
+ UInt32* ptr;
217
+ UChar* block;
218
+ UInt16* mtfv;
219
+ UChar* zbits;
220
+
221
+ /* for deciding when to use the fallback sorting algorithm */
222
+ Int32 workFactor;
223
+
224
+ /* run-length-encoding of the input */
225
+ UInt32 state_in_ch;
226
+ Int32 state_in_len;
227
+ BZ_RAND_DECLS;
228
+
229
+ /* input and output limits and current posns */
230
+ Int32 nblock;
231
+ Int32 nblockMAX;
232
+ Int32 numZ;
233
+ Int32 state_out_pos;
234
+
235
+ /* map of bytes used in block */
236
+ Int32 nInUse;
237
+ Bool inUse[256];
238
+ UChar unseqToSeq[256];
239
+
240
+ /* the buffer for bit stream creation */
241
+ UInt32 bsBuff;
242
+ Int32 bsLive;
243
+
244
+ /* block and combined CRCs */
245
+ UInt32 blockCRC;
246
+ UInt32 combinedCRC;
247
+
248
+ /* misc administratium */
249
+ Int32 verbosity;
250
+ Int32 blockNo;
251
+ Int32 blockSize100k;
252
+
253
+ /* stuff for coding the MTF values */
254
+ Int32 nMTF;
255
+ Int32 mtfFreq [BZ_MAX_ALPHA_SIZE];
256
+ UChar selector [BZ_MAX_SELECTORS];
257
+ UChar selectorMtf[BZ_MAX_SELECTORS];
258
+
259
+ UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
260
+ Int32 code [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
261
+ Int32 rfreq [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
262
+ /* second dimension: only 3 needed; 4 makes index calculations faster */
263
+ UInt32 len_pack[BZ_MAX_ALPHA_SIZE][4];
264
+
265
+ }
266
+ EState;
267
+
268
+
269
+
270
+ /*-- externs for compression. --*/
271
+
272
+ extern void
273
+ BZ2_blockSort ( EState* );
274
+
275
+ extern void
276
+ BZ2_compressBlock ( EState*, Bool );
277
+
278
+ extern void
279
+ BZ2_bsInitWrite ( EState* );
280
+
281
+ extern void
282
+ BZ2_hbAssignCodes ( Int32*, UChar*, Int32, Int32, Int32 );
283
+
284
+ extern void
285
+ BZ2_hbMakeCodeLengths ( UChar*, Int32*, Int32, Int32 );
286
+
287
+
288
+
289
+ /*-- states for decompression. --*/
290
+
291
+ #define BZ_X_IDLE 1
292
+ #define BZ_X_OUTPUT 2
293
+
294
+ #define BZ_X_MAGIC_1 10
295
+ #define BZ_X_MAGIC_2 11
296
+ #define BZ_X_MAGIC_3 12
297
+ #define BZ_X_MAGIC_4 13
298
+ #define BZ_X_BLKHDR_1 14
299
+ #define BZ_X_BLKHDR_2 15
300
+ #define BZ_X_BLKHDR_3 16
301
+ #define BZ_X_BLKHDR_4 17
302
+ #define BZ_X_BLKHDR_5 18
303
+ #define BZ_X_BLKHDR_6 19
304
+ #define BZ_X_BCRC_1 20
305
+ #define BZ_X_BCRC_2 21
306
+ #define BZ_X_BCRC_3 22
307
+ #define BZ_X_BCRC_4 23
308
+ #define BZ_X_RANDBIT 24
309
+ #define BZ_X_ORIGPTR_1 25
310
+ #define BZ_X_ORIGPTR_2 26
311
+ #define BZ_X_ORIGPTR_3 27
312
+ #define BZ_X_MAPPING_1 28
313
+ #define BZ_X_MAPPING_2 29
314
+ #define BZ_X_SELECTOR_1 30
315
+ #define BZ_X_SELECTOR_2 31
316
+ #define BZ_X_SELECTOR_3 32
317
+ #define BZ_X_CODING_1 33
318
+ #define BZ_X_CODING_2 34
319
+ #define BZ_X_CODING_3 35
320
+ #define BZ_X_MTF_1 36
321
+ #define BZ_X_MTF_2 37
322
+ #define BZ_X_MTF_3 38
323
+ #define BZ_X_MTF_4 39
324
+ #define BZ_X_MTF_5 40
325
+ #define BZ_X_MTF_6 41
326
+ #define BZ_X_ENDHDR_2 42
327
+ #define BZ_X_ENDHDR_3 43
328
+ #define BZ_X_ENDHDR_4 44
329
+ #define BZ_X_ENDHDR_5 45
330
+ #define BZ_X_ENDHDR_6 46
331
+ #define BZ_X_CCRC_1 47
332
+ #define BZ_X_CCRC_2 48
333
+ #define BZ_X_CCRC_3 49
334
+ #define BZ_X_CCRC_4 50
335
+
336
+
337
+
338
+ /*-- Constants for the fast MTF decoder. --*/
339
+
340
+ #define MTFA_SIZE 4096
341
+ #define MTFL_SIZE 16
342
+
343
+
344
+
345
+ /*-- Structure holding all the decompression-side stuff. --*/
346
+
347
+ typedef
348
+ struct {
349
+ /* pointer back to the struct bz_stream */
350
+ bz_stream* strm;
351
+
352
+ /* state indicator for this stream */
353
+ Int32 state;
354
+
355
+ /* for doing the final run-length decoding */
356
+ UChar state_out_ch;
357
+ Int32 state_out_len;
358
+ Bool blockRandomised;
359
+ BZ_RAND_DECLS;
360
+
361
+ /* the buffer for bit stream reading */
362
+ UInt32 bsBuff;
363
+ Int32 bsLive;
364
+
365
+ /* misc administratium */
366
+ Int32 blockSize100k;
367
+ Bool smallDecompress;
368
+ Int32 currBlockNo;
369
+ Int32 verbosity;
370
+
371
+ /* for undoing the Burrows-Wheeler transform */
372
+ Int32 origPtr;
373
+ UInt32 tPos;
374
+ Int32 k0;
375
+ Int32 unzftab[256];
376
+ Int32 nblock_used;
377
+ Int32 cftab[257];
378
+ Int32 cftabCopy[257];
379
+
380
+ /* for undoing the Burrows-Wheeler transform (FAST) */
381
+ UInt32 *tt;
382
+
383
+ /* for undoing the Burrows-Wheeler transform (SMALL) */
384
+ UInt16 *ll16;
385
+ UChar *ll4;
386
+
387
+ /* stored and calculated CRCs */
388
+ UInt32 storedBlockCRC;
389
+ UInt32 storedCombinedCRC;
390
+ UInt32 calculatedBlockCRC;
391
+ UInt32 calculatedCombinedCRC;
392
+
393
+ /* map of bytes used in block */
394
+ Int32 nInUse;
395
+ Bool inUse[256];
396
+ Bool inUse16[16];
397
+ UChar seqToUnseq[256];
398
+
399
+ /* for decoding the MTF values */
400
+ UChar mtfa [MTFA_SIZE];
401
+ Int32 mtfbase[256 / MTFL_SIZE];
402
+ UChar selector [BZ_MAX_SELECTORS];
403
+ UChar selectorMtf[BZ_MAX_SELECTORS];
404
+ UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
405
+
406
+ Int32 limit [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
407
+ Int32 base [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
408
+ Int32 perm [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
409
+ Int32 minLens[BZ_N_GROUPS];
410
+
411
+ /* save area for scalars in the main decompress code */
412
+ Int32 save_i;
413
+ Int32 save_j;
414
+ Int32 save_t;
415
+ Int32 save_alphaSize;
416
+ Int32 save_nGroups;
417
+ Int32 save_nSelectors;
418
+ Int32 save_EOB;
419
+ Int32 save_groupNo;
420
+ Int32 save_groupPos;
421
+ Int32 save_nextSym;
422
+ Int32 save_nblockMAX;
423
+ Int32 save_nblock;
424
+ Int32 save_es;
425
+ Int32 save_N;
426
+ Int32 save_curr;
427
+ Int32 save_zt;
428
+ Int32 save_zn;
429
+ Int32 save_zvec;
430
+ Int32 save_zj;
431
+ Int32 save_gSel;
432
+ Int32 save_gMinlen;
433
+ Int32* save_gLimit;
434
+ Int32* save_gBase;
435
+ Int32* save_gPerm;
436
+
437
+ }
438
+ DState;
439
+
440
+
441
+
442
+ /*-- Macros for decompression. --*/
443
+
444
+ #define BZ_GET_FAST(cccc) \
445
+ /* c_tPos is unsigned, hence test < 0 is pointless. */ \
446
+ if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \
447
+ s->tPos = s->tt[s->tPos]; \
448
+ cccc = (UChar)(s->tPos & 0xff); \
449
+ s->tPos >>= 8;
450
+
451
+ #define BZ_GET_FAST_C(cccc) \
452
+ /* c_tPos is unsigned, hence test < 0 is pointless. */ \
453
+ if (c_tPos >= (UInt32)100000 * (UInt32)ro_blockSize100k) return True; \
454
+ c_tPos = c_tt[c_tPos]; \
455
+ cccc = (UChar)(c_tPos & 0xff); \
456
+ c_tPos >>= 8;
457
+
458
+ #define SET_LL4(i,n) \
459
+ { if (((i) & 0x1) == 0) \
460
+ s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0xf0) | (n); else \
461
+ s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0x0f) | ((n) << 4); \
462
+ }
463
+
464
+ #define GET_LL4(i) \
465
+ ((((UInt32)(s->ll4[(i) >> 1])) >> (((i) << 2) & 0x4)) & 0xF)
466
+
467
+ #define SET_LL(i,n) \
468
+ { s->ll16[i] = (UInt16)(n & 0x0000ffff); \
469
+ SET_LL4(i, n >> 16); \
470
+ }
471
+
472
+ #define GET_LL(i) \
473
+ (((UInt32)s->ll16[i]) | (GET_LL4(i) << 16))
474
+
475
+ #define BZ_GET_SMALL(cccc) \
476
+ /* c_tPos is unsigned, hence test < 0 is pointless. */ \
477
+ if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \
478
+ cccc = BZ2_indexIntoF ( s->tPos, s->cftab ); \
479
+ s->tPos = GET_LL(s->tPos);
480
+
481
+
482
+ /*-- externs for decompression. --*/
483
+
484
+ extern Int32
485
+ BZ2_indexIntoF ( Int32, Int32* );
486
+
487
+ extern Int32
488
+ BZ2_decompress ( DState* );
489
+
490
+ extern void
491
+ BZ2_hbCreateDecodeTables ( Int32*, Int32*, Int32*, UChar*,
492
+ Int32, Int32, Int32 );
493
+
494
+
495
+ #endif
496
+
497
+
498
+ /*-- BZ_NO_STDIO seems to make NULL disappear on some platforms. --*/
499
+
500
+ #ifdef BZ_NO_STDIO
501
+ #ifndef NULL
502
+ #define NULL 0
503
+ #endif
504
+ #endif
505
+
506
+
507
+ /*-------------------------------------------------------------*/
508
+ /*--- end bzlib_private.h ---*/
509
+ /*-------------------------------------------------------------*/