google-protobuf 3.20.0.rc.1 → 3.20.0.rc.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of google-protobuf might be problematic. Click here for more details.

@@ -1,395 +0,0 @@
1
-
2
- /*
3
- * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
4
- *
5
- * Table 3-7. Well-Formed UTF-8 Byte Sequences
6
- *
7
- * +--------------------+------------+-------------+------------+-------------+
8
- * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
9
- * +--------------------+------------+-------------+------------+-------------+
10
- * | U+0000..U+007F | 00..7F | | | |
11
- * +--------------------+------------+-------------+------------+-------------+
12
- * | U+0080..U+07FF | C2..DF | 80..BF | | |
13
- * +--------------------+------------+-------------+------------+-------------+
14
- * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
15
- * +--------------------+------------+-------------+------------+-------------+
16
- * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
17
- * +--------------------+------------+-------------+------------+-------------+
18
- * | U+D000..U+D7FF | ED | 80..9F | 80..BF | |
19
- * +--------------------+------------+-------------+------------+-------------+
20
- * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
21
- * +--------------------+------------+-------------+------------+-------------+
22
- * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
23
- * +--------------------+------------+-------------+------------+-------------+
24
- * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
25
- * +--------------------+------------+-------------+------------+-------------+
26
- * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
27
- * +--------------------+------------+-------------+------------+-------------+
28
- */
29
-
30
- /* Return 0 - success, >0 - index(1 based) of first error char */
31
- int utf8_naive(const unsigned char* data, int len) {
32
- int err_pos = 1;
33
-
34
- while (len) {
35
- int bytes;
36
- const unsigned char byte1 = data[0];
37
-
38
- /* 00..7F */
39
- if (byte1 <= 0x7F) {
40
- bytes = 1;
41
- /* C2..DF, 80..BF */
42
- } else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF &&
43
- (signed char)data[1] <= (signed char)0xBF) {
44
- bytes = 2;
45
- } else if (len >= 3) {
46
- const unsigned char byte2 = data[1];
47
-
48
- /* Is byte2, byte3 between 0x80 ~ 0xBF */
49
- const int byte2_ok = (signed char)byte2 <= (signed char)0xBF;
50
- const int byte3_ok = (signed char)data[2] <= (signed char)0xBF;
51
-
52
- if (byte2_ok && byte3_ok &&
53
- /* E0, A0..BF, 80..BF */
54
- ((byte1 == 0xE0 && byte2 >= 0xA0) ||
55
- /* E1..EC, 80..BF, 80..BF */
56
- (byte1 >= 0xE1 && byte1 <= 0xEC) ||
57
- /* ED, 80..9F, 80..BF */
58
- (byte1 == 0xED && byte2 <= 0x9F) ||
59
- /* EE..EF, 80..BF, 80..BF */
60
- (byte1 >= 0xEE && byte1 <= 0xEF))) {
61
- bytes = 3;
62
- } else if (len >= 4) {
63
- /* Is byte4 between 0x80 ~ 0xBF */
64
- const int byte4_ok = (signed char)data[3] <= (signed char)0xBF;
65
-
66
- if (byte2_ok && byte3_ok && byte4_ok &&
67
- /* F0, 90..BF, 80..BF, 80..BF */
68
- ((byte1 == 0xF0 && byte2 >= 0x90) ||
69
- /* F1..F3, 80..BF, 80..BF, 80..BF */
70
- (byte1 >= 0xF1 && byte1 <= 0xF3) ||
71
- /* F4, 80..8F, 80..BF, 80..BF */
72
- (byte1 == 0xF4 && byte2 <= 0x8F))) {
73
- bytes = 4;
74
- } else {
75
- return err_pos;
76
- }
77
- } else {
78
- return err_pos;
79
- }
80
- } else {
81
- return err_pos;
82
- }
83
-
84
- len -= bytes;
85
- err_pos += bytes;
86
- data += bytes;
87
- }
88
-
89
- return 0;
90
- }
91
-
92
- #ifdef __SSE4_1__
93
-
94
- #include <stdint.h>
95
- #include <stdio.h>
96
- #include <x86intrin.h>
97
-
98
- int utf8_naive(const unsigned char* data, int len);
99
-
100
- static const int8_t _first_len_tbl[] = {
101
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
102
- };
103
-
104
- static const int8_t _first_range_tbl[] = {
105
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
106
- };
107
-
108
- static const int8_t _range_min_tbl[] = {
109
- 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
110
- 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F,
111
- };
112
- static const int8_t _range_max_tbl[] = {
113
- 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
114
- 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
115
- };
116
-
117
- static const int8_t _df_ee_tbl[] = {
118
- 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0,
119
- };
120
- static const int8_t _ef_fe_tbl[] = {
121
- 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
122
- };
123
-
124
- /* Return 0 on success, -1 on error */
125
- int utf8_range2(const unsigned char* data, int len) {
126
- if (len >= 32) {
127
- __m128i prev_input = _mm_set1_epi8(0);
128
- __m128i prev_first_len = _mm_set1_epi8(0);
129
-
130
- const __m128i first_len_tbl =
131
- _mm_loadu_si128((const __m128i*)_first_len_tbl);
132
- const __m128i first_range_tbl =
133
- _mm_loadu_si128((const __m128i*)_first_range_tbl);
134
- const __m128i range_min_tbl =
135
- _mm_loadu_si128((const __m128i*)_range_min_tbl);
136
- const __m128i range_max_tbl =
137
- _mm_loadu_si128((const __m128i*)_range_max_tbl);
138
- const __m128i df_ee_tbl = _mm_loadu_si128((const __m128i*)_df_ee_tbl);
139
- const __m128i ef_fe_tbl = _mm_loadu_si128((const __m128i*)_ef_fe_tbl);
140
-
141
- __m128i error = _mm_set1_epi8(0);
142
-
143
- while (len >= 32) {
144
- /***************************** block 1 ****************************/
145
- const __m128i input_a = _mm_loadu_si128((const __m128i*)data);
146
-
147
- __m128i high_nibbles =
148
- _mm_and_si128(_mm_srli_epi16(input_a, 4), _mm_set1_epi8(0x0F));
149
-
150
- __m128i first_len_a = _mm_shuffle_epi8(first_len_tbl, high_nibbles);
151
-
152
- __m128i range_a = _mm_shuffle_epi8(first_range_tbl, high_nibbles);
153
-
154
- range_a = _mm_or_si128(range_a,
155
- _mm_alignr_epi8(first_len_a, prev_first_len, 15));
156
-
157
- __m128i tmp;
158
- tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 14);
159
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1));
160
- range_a = _mm_or_si128(range_a, tmp);
161
-
162
- tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 13);
163
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2));
164
- range_a = _mm_or_si128(range_a, tmp);
165
-
166
- __m128i shift1, pos, range2;
167
- shift1 = _mm_alignr_epi8(input_a, prev_input, 15);
168
- pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF));
169
- tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0));
170
- range2 = _mm_shuffle_epi8(df_ee_tbl, tmp);
171
- tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70));
172
- range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp));
173
-
174
- range_a = _mm_add_epi8(range_a, range2);
175
-
176
- __m128i minv = _mm_shuffle_epi8(range_min_tbl, range_a);
177
- __m128i maxv = _mm_shuffle_epi8(range_max_tbl, range_a);
178
-
179
- tmp = _mm_or_si128(_mm_cmplt_epi8(input_a, minv),
180
- _mm_cmpgt_epi8(input_a, maxv));
181
- error = _mm_or_si128(error, tmp);
182
-
183
- /***************************** block 2 ****************************/
184
- const __m128i input_b = _mm_loadu_si128((const __m128i*)(data + 16));
185
-
186
- high_nibbles =
187
- _mm_and_si128(_mm_srli_epi16(input_b, 4), _mm_set1_epi8(0x0F));
188
-
189
- __m128i first_len_b = _mm_shuffle_epi8(first_len_tbl, high_nibbles);
190
-
191
- __m128i range_b = _mm_shuffle_epi8(first_range_tbl, high_nibbles);
192
-
193
- range_b =
194
- _mm_or_si128(range_b, _mm_alignr_epi8(first_len_b, first_len_a, 15));
195
-
196
- tmp = _mm_alignr_epi8(first_len_b, first_len_a, 14);
197
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1));
198
- range_b = _mm_or_si128(range_b, tmp);
199
-
200
- tmp = _mm_alignr_epi8(first_len_b, first_len_a, 13);
201
- tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2));
202
- range_b = _mm_or_si128(range_b, tmp);
203
-
204
- shift1 = _mm_alignr_epi8(input_b, input_a, 15);
205
- pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF));
206
- tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0));
207
- range2 = _mm_shuffle_epi8(df_ee_tbl, tmp);
208
- tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70));
209
- range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp));
210
-
211
- range_b = _mm_add_epi8(range_b, range2);
212
-
213
- minv = _mm_shuffle_epi8(range_min_tbl, range_b);
214
- maxv = _mm_shuffle_epi8(range_max_tbl, range_b);
215
-
216
- tmp = _mm_or_si128(_mm_cmplt_epi8(input_b, minv),
217
- _mm_cmpgt_epi8(input_b, maxv));
218
- error = _mm_or_si128(error, tmp);
219
-
220
- /************************ next iteration **************************/
221
- prev_input = input_b;
222
- prev_first_len = first_len_b;
223
-
224
- data += 32;
225
- len -= 32;
226
- }
227
-
228
- if (!_mm_testz_si128(error, error)) return -1;
229
-
230
- int32_t token4 = _mm_extract_epi32(prev_input, 3);
231
- const int8_t* token = (const int8_t*)&token4;
232
- int lookahead = 0;
233
- if (token[3] > (int8_t)0xBF)
234
- lookahead = 1;
235
- else if (token[2] > (int8_t)0xBF)
236
- lookahead = 2;
237
- else if (token[1] > (int8_t)0xBF)
238
- lookahead = 3;
239
-
240
- data -= lookahead;
241
- len += lookahead;
242
- }
243
-
244
- return utf8_naive(data, len);
245
- }
246
-
247
- #endif
248
-
249
- #ifdef __ARM_NEON
250
-
251
- #include <arm_neon.h>
252
- #include <stdint.h>
253
- #include <stdio.h>
254
-
255
- int utf8_naive(const unsigned char* data, int len);
256
-
257
- static const uint8_t _first_len_tbl[] = {
258
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
259
- };
260
-
261
- static const uint8_t _first_range_tbl[] = {
262
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
263
- };
264
-
265
- static const uint8_t _range_min_tbl[] = {
266
- 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
267
- 0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
268
- };
269
- static const uint8_t _range_max_tbl[] = {
270
- 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
271
- 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
272
- };
273
-
274
- static const uint8_t _range_adjust_tbl[] = {
275
- 2, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0,
276
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0,
277
- };
278
-
279
- /* Return 0 on success, -1 on error */
280
- int utf8_range2(const unsigned char* data, int len) {
281
- if (len >= 32) {
282
- uint8x16_t prev_input = vdupq_n_u8(0);
283
- uint8x16_t prev_first_len = vdupq_n_u8(0);
284
-
285
- const uint8x16_t first_len_tbl = vld1q_u8(_first_len_tbl);
286
- const uint8x16_t first_range_tbl = vld1q_u8(_first_range_tbl);
287
- const uint8x16_t range_min_tbl = vld1q_u8(_range_min_tbl);
288
- const uint8x16_t range_max_tbl = vld1q_u8(_range_max_tbl);
289
- const uint8x16x2_t range_adjust_tbl = vld2q_u8(_range_adjust_tbl);
290
-
291
- const uint8x16_t const_1 = vdupq_n_u8(1);
292
- const uint8x16_t const_2 = vdupq_n_u8(2);
293
- const uint8x16_t const_e0 = vdupq_n_u8(0xE0);
294
-
295
- uint8x16_t error1 = vdupq_n_u8(0);
296
- uint8x16_t error2 = vdupq_n_u8(0);
297
- uint8x16_t error3 = vdupq_n_u8(0);
298
- uint8x16_t error4 = vdupq_n_u8(0);
299
-
300
- while (len >= 32) {
301
- /******************* two blocks interleaved **********************/
302
-
303
- #if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 8)
304
- /* gcc doesn't support vldq1_u8_x2 until version 8 */
305
- const uint8x16_t input_a = vld1q_u8(data);
306
- const uint8x16_t input_b = vld1q_u8(data + 16);
307
- #else
308
- /* Forces a double load on Clang */
309
- const uint8x16x2_t input_pair = vld1q_u8_x2(data);
310
- const uint8x16_t input_a = input_pair.val[0];
311
- const uint8x16_t input_b = input_pair.val[1];
312
- #endif
313
-
314
- const uint8x16_t high_nibbles_a = vshrq_n_u8(input_a, 4);
315
- const uint8x16_t high_nibbles_b = vshrq_n_u8(input_b, 4);
316
-
317
- const uint8x16_t first_len_a = vqtbl1q_u8(first_len_tbl, high_nibbles_a);
318
- const uint8x16_t first_len_b = vqtbl1q_u8(first_len_tbl, high_nibbles_b);
319
-
320
- uint8x16_t range_a = vqtbl1q_u8(first_range_tbl, high_nibbles_a);
321
- uint8x16_t range_b = vqtbl1q_u8(first_range_tbl, high_nibbles_b);
322
-
323
- range_a = vorrq_u8(range_a, vextq_u8(prev_first_len, first_len_a, 15));
324
- range_b = vorrq_u8(range_b, vextq_u8(first_len_a, first_len_b, 15));
325
-
326
- uint8x16_t tmp1_a, tmp2_a, tmp1_b, tmp2_b;
327
- tmp1_a = vextq_u8(prev_first_len, first_len_a, 14);
328
- tmp1_a = vqsubq_u8(tmp1_a, const_1);
329
- range_a = vorrq_u8(range_a, tmp1_a);
330
-
331
- tmp1_b = vextq_u8(first_len_a, first_len_b, 14);
332
- tmp1_b = vqsubq_u8(tmp1_b, const_1);
333
- range_b = vorrq_u8(range_b, tmp1_b);
334
-
335
- tmp2_a = vextq_u8(prev_first_len, first_len_a, 13);
336
- tmp2_a = vqsubq_u8(tmp2_a, const_2);
337
- range_a = vorrq_u8(range_a, tmp2_a);
338
-
339
- tmp2_b = vextq_u8(first_len_a, first_len_b, 13);
340
- tmp2_b = vqsubq_u8(tmp2_b, const_2);
341
- range_b = vorrq_u8(range_b, tmp2_b);
342
-
343
- uint8x16_t shift1_a = vextq_u8(prev_input, input_a, 15);
344
- uint8x16_t pos_a = vsubq_u8(shift1_a, const_e0);
345
- range_a = vaddq_u8(range_a, vqtbl2q_u8(range_adjust_tbl, pos_a));
346
-
347
- uint8x16_t shift1_b = vextq_u8(input_a, input_b, 15);
348
- uint8x16_t pos_b = vsubq_u8(shift1_b, const_e0);
349
- range_b = vaddq_u8(range_b, vqtbl2q_u8(range_adjust_tbl, pos_b));
350
-
351
- uint8x16_t minv_a = vqtbl1q_u8(range_min_tbl, range_a);
352
- uint8x16_t maxv_a = vqtbl1q_u8(range_max_tbl, range_a);
353
-
354
- uint8x16_t minv_b = vqtbl1q_u8(range_min_tbl, range_b);
355
- uint8x16_t maxv_b = vqtbl1q_u8(range_max_tbl, range_b);
356
-
357
- error1 = vorrq_u8(error1, vcltq_u8(input_a, minv_a));
358
- error2 = vorrq_u8(error2, vcgtq_u8(input_a, maxv_a));
359
-
360
- error3 = vorrq_u8(error3, vcltq_u8(input_b, minv_b));
361
- error4 = vorrq_u8(error4, vcgtq_u8(input_b, maxv_b));
362
-
363
- /************************ next iteration *************************/
364
- prev_input = input_b;
365
- prev_first_len = first_len_b;
366
-
367
- data += 32;
368
- len -= 32;
369
- }
370
- error1 = vorrq_u8(error1, error2);
371
- error1 = vorrq_u8(error1, error3);
372
- error1 = vorrq_u8(error1, error4);
373
-
374
- if (vmaxvq_u8(error1)) return -1;
375
-
376
- uint32_t token4;
377
- vst1q_lane_u32(&token4, vreinterpretq_u32_u8(prev_input), 3);
378
-
379
- const int8_t* token = (const int8_t*)&token4;
380
- int lookahead = 0;
381
- if (token[3] > (int8_t)0xBF)
382
- lookahead = 1;
383
- else if (token[2] > (int8_t)0xBF)
384
- lookahead = 2;
385
- else if (token[1] > (int8_t)0xBF)
386
- lookahead = 3;
387
-
388
- data -= lookahead;
389
- len += lookahead;
390
- }
391
-
392
- return utf8_naive(data, len);
393
- }
394
-
395
- #endif