is_it_utf8 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 5bcf193a3583685668ac42302a9b3f39017c529b2019c3936cad1156c8e88276
4
+ data.tar.gz: 46d46ef5d59cacd22c2eafbd5b01de46f074e2c44f532e65b57c4c7c7fd46428
5
+ SHA512:
6
+ metadata.gz: 2f9bb50d4b69e7a11bd24ed97af8aa2dd2c15429d42cabf108096bac91f8ce214422fd79b99d01c1b93e28c051330fd9efb139e90a5e5f625b14b73a68dd5bc6
7
+ data.tar.gz: 2939e5fc01aa4b5a8b657e1869c5af1f19dba99ba25cea4f9adcc1f92e379185c709767c3b1d7beacc57ad4b16243aa2c508bee0fdac77e3e8f765e83498895c
@@ -0,0 +1,8 @@
1
+ require 'mkmf'
2
+
3
+ dir_config('is_it_utf8')
4
+ if find_header 'simdutf8check.h', File.expand_path(File.dirname(__FILE__))
5
+ create_makefile('is_it_utf8')
6
+ else
7
+ raise "Could not find simdutf8check.h"
8
+ end
@@ -0,0 +1,35 @@
1
+ #include <stdlib.h>
2
+ #include <ruby.h>
3
+ #include <ruby/encoding.h>
4
+ #include "simdutf8check.h"
5
+
6
+ static VALUE m_IsItUtf8;
7
+
8
+ static VALUE
9
+ is_it_utf8(int argc, VALUE* argv, VALUE self)
10
+ {
11
+ VALUE input;
12
+ const unsigned char* src;
13
+ rb_scan_args(argc, argv, "1", &input);
14
+
15
+ if (TYPE(input) != T_STRING) {
16
+ rb_raise(rb_eTypeError, "expected a String");
17
+ }
18
+
19
+ src = (unsigned char*) StringValueCStr(input);
20
+ bool is_valid = validate_utf8_fast((const char *) src, RSTRING_LEN(input));
21
+
22
+ if(is_valid) {
23
+ return Qtrue;
24
+ } else {
25
+ return Qfalse;
26
+ }
27
+ }
28
+
29
+
30
+ void
31
+ Init_is_it_utf8() {
32
+ m_IsItUtf8 = rb_define_module("IsItUtf8");
33
+
34
+ rb_define_module_function(m_IsItUtf8, "valid?", is_it_utf8, -1);
35
+ }
@@ -0,0 +1,458 @@
1
+
2
+ #ifndef SIMDUTF8CHECK_H
3
+ #define SIMDUTF8CHECK_H
4
+ #include <stdbool.h>
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+ #include <string.h>
8
+ #include <x86intrin.h>
9
+ /*
10
+ * legal utf-8 byte sequence
11
+ * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
12
+ *
13
+ * Code Points 1st 2s 3s 4s
14
+ * U+0000..U+007F 00..7F
15
+ * U+0080..U+07FF C2..DF 80..BF
16
+ * U+0800..U+0FFF E0 A0..BF 80..BF
17
+ * U+1000..U+CFFF E1..EC 80..BF 80..BF
18
+ * U+D000..U+D7FF ED 80..9F 80..BF
19
+ * U+E000..U+FFFF EE..EF 80..BF 80..BF
20
+ * U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
21
+ * U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
22
+ * U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
23
+ *
24
+ */
25
+
26
+ // all byte values must be no larger than 0xF4
27
+ static inline void checkSmallerThan0xF4(__m128i current_bytes,
28
+ __m128i *has_error) {
29
+ // unsigned, saturates to 0 below max
30
+ *has_error = _mm_or_si128(*has_error,
31
+ _mm_subs_epu8(current_bytes, _mm_set1_epi8(0xF4)));
32
+ }
33
+
34
+ static inline __m128i continuationLengths(__m128i high_nibbles) {
35
+ return _mm_shuffle_epi8(
36
+ _mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
37
+ 0, 0, 0, 0, // 10xx (continuation)
38
+ 2, 2, // 110x
39
+ 3, // 1110
40
+ 4), // 1111, next should be 0 (not checked here)
41
+ high_nibbles);
42
+ }
43
+
44
+ static inline __m128i carryContinuations(__m128i initial_lengths,
45
+ __m128i previous_carries) {
46
+
47
+ __m128i right1 =
48
+ _mm_subs_epu8(_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1),
49
+ _mm_set1_epi8(1));
50
+ __m128i sum = _mm_add_epi8(initial_lengths, right1);
51
+
52
+ __m128i right2 = _mm_subs_epu8(_mm_alignr_epi8(sum, previous_carries, 16 - 2),
53
+ _mm_set1_epi8(2));
54
+ return _mm_add_epi8(sum, right2);
55
+ }
56
+
57
+ static inline void checkContinuations(__m128i initial_lengths, __m128i carries,
58
+ __m128i *has_error) {
59
+
60
+ // overlap || underlap
61
+ // carry > length && length > 0 || !(carry > length) && !(length > 0)
62
+ // (carries > length) == (lengths > 0)
63
+ __m128i overunder =
64
+ _mm_cmpeq_epi8(_mm_cmpgt_epi8(carries, initial_lengths),
65
+ _mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128()));
66
+
67
+ *has_error = _mm_or_si128(*has_error, overunder);
68
+ }
69
+
70
+ // when 0xED is found, next byte must be no larger than 0x9F
71
+ // when 0xF4 is found, next byte must be no larger than 0x8F
72
+ // next byte must be continuation, ie sign bit is set, so signed < is ok
73
+ static inline void checkFirstContinuationMax(__m128i current_bytes,
74
+ __m128i off1_current_bytes,
75
+ __m128i *has_error) {
76
+ __m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xED));
77
+ __m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4));
78
+
79
+ __m128i badfollowED =
80
+ _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9F)), maskED);
81
+ __m128i badfollowF4 =
82
+ _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8F)), maskF4);
83
+
84
+ *has_error = _mm_or_si128(*has_error, _mm_or_si128(badfollowED, badfollowF4));
85
+ }
86
+
87
+ // map off1_hibits => error condition
88
+ // hibits off1 cur
89
+ // C => < C2 && true
90
+ // E => < E1 && < A0
91
+ // F => < F1 && < 90
92
+ // else false && false
93
+ static inline void checkOverlong(__m128i current_bytes,
94
+ __m128i off1_current_bytes, __m128i hibits,
95
+ __m128i previous_hibits, __m128i *has_error) {
96
+ __m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1);
97
+ __m128i initial_mins = _mm_shuffle_epi8(
98
+ _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128,
99
+ -128, -128, // 10xx => false
100
+ 0xC2, -128, // 110x
101
+ 0xE1, // 1110
102
+ 0xF1),
103
+ off1_hibits);
104
+
105
+ __m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes);
106
+
107
+ __m128i second_mins = _mm_shuffle_epi8(
108
+ _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128,
109
+ -128, -128, // 10xx => false
110
+ 127, 127, // 110x => true
111
+ 0xA0, // 1110
112
+ 0x90),
113
+ off1_hibits);
114
+ __m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes);
115
+ *has_error =
116
+ _mm_or_si128(*has_error, _mm_and_si128(initial_under, second_under));
117
+ }
118
+
119
+ struct processed_utf_bytes {
120
+ __m128i rawbytes;
121
+ __m128i high_nibbles;
122
+ __m128i carried_continuations;
123
+ };
124
+
125
+ static inline void count_nibbles(__m128i bytes,
126
+ struct processed_utf_bytes *answer) {
127
+ answer->rawbytes = bytes;
128
+ answer->high_nibbles =
129
+ _mm_and_si128(_mm_srli_epi16(bytes, 4), _mm_set1_epi8(0x0F));
130
+ }
131
+
132
+ // check whether the current bytes are valid UTF-8
133
+ // at the end of the function, previous gets updated
134
+ static struct processed_utf_bytes
135
+ checkUTF8Bytes(__m128i current_bytes, struct processed_utf_bytes *previous,
136
+ __m128i *has_error) {
137
+ struct processed_utf_bytes pb;
138
+ count_nibbles(current_bytes, &pb);
139
+
140
+ checkSmallerThan0xF4(current_bytes, has_error);
141
+
142
+ __m128i initial_lengths = continuationLengths(pb.high_nibbles);
143
+
144
+ pb.carried_continuations =
145
+ carryContinuations(initial_lengths, previous->carried_continuations);
146
+
147
+ checkContinuations(initial_lengths, pb.carried_continuations, has_error);
148
+
149
+ __m128i off1_current_bytes =
150
+ _mm_alignr_epi8(pb.rawbytes, previous->rawbytes, 16 - 1);
151
+ checkFirstContinuationMax(current_bytes, off1_current_bytes, has_error);
152
+
153
+ checkOverlong(current_bytes, off1_current_bytes, pb.high_nibbles,
154
+ previous->high_nibbles, has_error);
155
+ return pb;
156
+ }
157
+
158
+ static bool validate_utf8_fast(const char *src, size_t len) {
159
+ size_t i = 0;
160
+ __m128i has_error = _mm_setzero_si128();
161
+ struct processed_utf_bytes previous = {.rawbytes = _mm_setzero_si128(),
162
+ .high_nibbles = _mm_setzero_si128(),
163
+ .carried_continuations =
164
+ _mm_setzero_si128()};
165
+ if (len >= 16) {
166
+ for (; i <= len - 16; i += 16) {
167
+ __m128i current_bytes = _mm_loadu_si128((const __m128i *)(src + i));
168
+ previous = checkUTF8Bytes(current_bytes, &previous, &has_error);
169
+ }
170
+ }
171
+
172
+ // last part
173
+ if (i < len) {
174
+ char buffer[16];
175
+ memset(buffer, 0, 16);
176
+ memcpy(buffer, src + i, len - i);
177
+ __m128i current_bytes = _mm_loadu_si128((const __m128i *)(buffer));
178
+ previous = checkUTF8Bytes(current_bytes, &previous, &has_error);
179
+ } else {
180
+ has_error =
181
+ _mm_or_si128(_mm_cmpgt_epi8(previous.carried_continuations,
182
+ _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
183
+ 9, 9, 9, 9, 9, 1)),
184
+ has_error);
185
+ }
186
+
187
+ return _mm_testz_si128(has_error, has_error);
188
+ }
189
+
190
+ #ifdef __AVX2__
191
+
192
+ /*****************************/
193
+ static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) {
194
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15);
195
+ }
196
+
197
+ static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b) {
198
+ return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 14);
199
+ }
200
+
201
+ // all byte values must be no larger than 0xF4
202
+ static inline void avxcheckSmallerThan0xF4(__m256i current_bytes,
203
+ __m256i *has_error) {
204
+ // unsigned, saturates to 0 below max
205
+ *has_error = _mm256_or_si256(
206
+ *has_error, _mm256_subs_epu8(current_bytes, _mm256_set1_epi8(0xF4)));
207
+ }
208
+
209
+ static inline __m256i avxcontinuationLengths(__m256i high_nibbles) {
210
+ return _mm256_shuffle_epi8(
211
+ _mm256_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
212
+ 0, 0, 0, 0, // 10xx (continuation)
213
+ 2, 2, // 110x
214
+ 3, // 1110
215
+ 4, // 1111, next should be 0 (not checked here)
216
+ 1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
217
+ 0, 0, 0, 0, // 10xx (continuation)
218
+ 2, 2, // 110x
219
+ 3, // 1110
220
+ 4 // 1111, next should be 0 (not checked here)
221
+ ),
222
+ high_nibbles);
223
+ }
224
+
225
+ static inline __m256i avxcarryContinuations(__m256i initial_lengths,
226
+ __m256i previous_carries) {
227
+
228
+ __m256i right1 = _mm256_subs_epu8(
229
+ push_last_byte_of_a_to_b(previous_carries, initial_lengths),
230
+ _mm256_set1_epi8(1));
231
+ __m256i sum = _mm256_add_epi8(initial_lengths, right1);
232
+
233
+ __m256i right2 = _mm256_subs_epu8(
234
+ push_last_2bytes_of_a_to_b(previous_carries, sum), _mm256_set1_epi8(2));
235
+ return _mm256_add_epi8(sum, right2);
236
+ }
237
+
238
+ static inline void avxcheckContinuations(__m256i initial_lengths,
239
+ __m256i carries, __m256i *has_error) {
240
+
241
+ // overlap || underlap
242
+ // carry > length && length > 0 || !(carry > length) && !(length > 0)
243
+ // (carries > length) == (lengths > 0)
244
+ __m256i overunder = _mm256_cmpeq_epi8(
245
+ _mm256_cmpgt_epi8(carries, initial_lengths),
246
+ _mm256_cmpgt_epi8(initial_lengths, _mm256_setzero_si256()));
247
+
248
+ *has_error = _mm256_or_si256(*has_error, overunder);
249
+ }
250
+
251
+ // when 0xED is found, next byte must be no larger than 0x9F
252
+ // when 0xF4 is found, next byte must be no larger than 0x8F
253
+ // next byte must be continuation, ie sign bit is set, so signed < is ok
254
+ static inline void avxcheckFirstContinuationMax(__m256i current_bytes,
255
+ __m256i off1_current_bytes,
256
+ __m256i *has_error) {
257
+ __m256i maskED =
258
+ _mm256_cmpeq_epi8(off1_current_bytes, _mm256_set1_epi8(0xED));
259
+ __m256i maskF4 =
260
+ _mm256_cmpeq_epi8(off1_current_bytes, _mm256_set1_epi8(0xF4));
261
+
262
+ __m256i badfollowED = _mm256_and_si256(
263
+ _mm256_cmpgt_epi8(current_bytes, _mm256_set1_epi8(0x9F)), maskED);
264
+ __m256i badfollowF4 = _mm256_and_si256(
265
+ _mm256_cmpgt_epi8(current_bytes, _mm256_set1_epi8(0x8F)), maskF4);
266
+
267
+ *has_error =
268
+ _mm256_or_si256(*has_error, _mm256_or_si256(badfollowED, badfollowF4));
269
+ }
270
+
271
+ // map off1_hibits => error condition
272
+ // hibits off1 cur
273
+ // C => < C2 && true
274
+ // E => < E1 && < A0
275
+ // F => < F1 && < 90
276
+ // else false && false
277
+ static inline void avxcheckOverlong(__m256i current_bytes,
278
+ __m256i off1_current_bytes, __m256i hibits,
279
+ __m256i previous_hibits,
280
+ __m256i *has_error) {
281
+ __m256i off1_hibits = push_last_byte_of_a_to_b(previous_hibits, hibits);
282
+ __m256i initial_mins = _mm256_shuffle_epi8(
283
+ _mm256_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128,
284
+ -128, -128, -128, // 10xx => false
285
+ 0xC2, -128, // 110x
286
+ 0xE1, // 1110
287
+ 0xF1, -128, -128, -128, -128, -128, -128, -128, -128,
288
+ -128, -128, -128, -128, // 10xx => false
289
+ 0xC2, -128, // 110x
290
+ 0xE1, // 1110
291
+ 0xF1),
292
+ off1_hibits);
293
+
294
+ __m256i initial_under = _mm256_cmpgt_epi8(initial_mins, off1_current_bytes);
295
+
296
+ __m256i second_mins = _mm256_shuffle_epi8(
297
+ _mm256_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128,
298
+ -128, -128, -128, // 10xx => false
299
+ 127, 127, // 110x => true
300
+ 0xA0, // 1110
301
+ 0x90, -128, -128, -128, -128, -128, -128, -128, -128,
302
+ -128, -128, -128, -128, // 10xx => false
303
+ 127, 127, // 110x => true
304
+ 0xA0, // 1110
305
+ 0x90),
306
+ off1_hibits);
307
+ __m256i second_under = _mm256_cmpgt_epi8(second_mins, current_bytes);
308
+ *has_error = _mm256_or_si256(*has_error,
309
+ _mm256_and_si256(initial_under, second_under));
310
+ }
311
+
312
+ struct avx_processed_utf_bytes {
313
+ __m256i rawbytes;
314
+ __m256i high_nibbles;
315
+ __m256i carried_continuations;
316
+ };
317
+
318
+ static inline void avx_count_nibbles(__m256i bytes,
319
+ struct avx_processed_utf_bytes *answer) {
320
+ answer->rawbytes = bytes;
321
+ answer->high_nibbles =
322
+ _mm256_and_si256(_mm256_srli_epi16(bytes, 4), _mm256_set1_epi8(0x0F));
323
+ }
324
+
325
+ // check whether the current bytes are valid UTF-8
326
+ // at the end of the function, previous gets updated
327
+ static struct avx_processed_utf_bytes
328
+ avxcheckUTF8Bytes(__m256i current_bytes,
329
+ struct avx_processed_utf_bytes *previous,
330
+ __m256i *has_error) {
331
+ struct avx_processed_utf_bytes pb;
332
+ avx_count_nibbles(current_bytes, &pb);
333
+
334
+ avxcheckSmallerThan0xF4(current_bytes, has_error);
335
+
336
+ __m256i initial_lengths = avxcontinuationLengths(pb.high_nibbles);
337
+
338
+ pb.carried_continuations =
339
+ avxcarryContinuations(initial_lengths, previous->carried_continuations);
340
+
341
+ avxcheckContinuations(initial_lengths, pb.carried_continuations, has_error);
342
+
343
+ __m256i off1_current_bytes =
344
+ push_last_byte_of_a_to_b(previous->rawbytes, pb.rawbytes);
345
+ avxcheckFirstContinuationMax(current_bytes, off1_current_bytes, has_error);
346
+
347
+ avxcheckOverlong(current_bytes, off1_current_bytes, pb.high_nibbles,
348
+ previous->high_nibbles, has_error);
349
+ return pb;
350
+ }
351
+
352
+ // check whether the current bytes are valid UTF-8
353
+ // at the end of the function, previous gets updated
354
+ static struct avx_processed_utf_bytes
355
+ avxcheckUTF8Bytes_asciipath(__m256i current_bytes,
356
+ struct avx_processed_utf_bytes *previous,
357
+ __m256i *has_error) {
358
+ if (_mm256_testz_si256(current_bytes,
359
+ _mm256_set1_epi8(0x80))) { // fast ascii path
360
+ *has_error = _mm256_or_si256(
361
+ _mm256_cmpgt_epi8(previous->carried_continuations,
362
+ _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
363
+ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
364
+ 9, 9, 9, 9, 9, 9, 9, 1)),
365
+ *has_error);
366
+ return *previous;
367
+ }
368
+
369
+ struct avx_processed_utf_bytes pb;
370
+ avx_count_nibbles(current_bytes, &pb);
371
+
372
+ avxcheckSmallerThan0xF4(current_bytes, has_error);
373
+
374
+ __m256i initial_lengths = avxcontinuationLengths(pb.high_nibbles);
375
+
376
+ pb.carried_continuations =
377
+ avxcarryContinuations(initial_lengths, previous->carried_continuations);
378
+
379
+ avxcheckContinuations(initial_lengths, pb.carried_continuations, has_error);
380
+
381
+ __m256i off1_current_bytes =
382
+ push_last_byte_of_a_to_b(previous->rawbytes, pb.rawbytes);
383
+ avxcheckFirstContinuationMax(current_bytes, off1_current_bytes, has_error);
384
+
385
+ avxcheckOverlong(current_bytes, off1_current_bytes, pb.high_nibbles,
386
+ previous->high_nibbles, has_error);
387
+ return pb;
388
+ }
389
+
390
+ static bool validate_utf8_fast_avx_asciipath(const char *src, size_t len) {
391
+ size_t i = 0;
392
+ __m256i has_error = _mm256_setzero_si256();
393
+ struct avx_processed_utf_bytes previous = {
394
+ .rawbytes = _mm256_setzero_si256(),
395
+ .high_nibbles = _mm256_setzero_si256(),
396
+ .carried_continuations = _mm256_setzero_si256()};
397
+ if (len >= 32) {
398
+ for (; i <= len - 32; i += 32) {
399
+ __m256i current_bytes = _mm256_loadu_si256((const __m256i *)(src + i));
400
+ previous =
401
+ avxcheckUTF8Bytes_asciipath(current_bytes, &previous, &has_error);
402
+ }
403
+ }
404
+
405
+ // last part
406
+ if (i < len) {
407
+ char buffer[32];
408
+ memset(buffer, 0, 32);
409
+ memcpy(buffer, src + i, len - i);
410
+ __m256i current_bytes = _mm256_loadu_si256((const __m256i *)(buffer));
411
+ previous = avxcheckUTF8Bytes(current_bytes, &previous, &has_error);
412
+ } else {
413
+ has_error = _mm256_or_si256(
414
+ _mm256_cmpgt_epi8(previous.carried_continuations,
415
+ _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
416
+ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
417
+ 9, 9, 9, 9, 9, 9, 9, 1)),
418
+ has_error);
419
+ }
420
+
421
+ return _mm256_testz_si256(has_error, has_error);
422
+ }
423
+
424
+ static bool validate_utf8_fast_avx(const char *src, size_t len) {
425
+ size_t i = 0;
426
+ __m256i has_error = _mm256_setzero_si256();
427
+ struct avx_processed_utf_bytes previous = {
428
+ .rawbytes = _mm256_setzero_si256(),
429
+ .high_nibbles = _mm256_setzero_si256(),
430
+ .carried_continuations = _mm256_setzero_si256()};
431
+ if (len >= 32) {
432
+ for (; i <= len - 32; i += 32) {
433
+ __m256i current_bytes = _mm256_loadu_si256((const __m256i *)(src + i));
434
+ previous = avxcheckUTF8Bytes(current_bytes, &previous, &has_error);
435
+ }
436
+ }
437
+
438
+ // last part
439
+ if (i < len) {
440
+ char buffer[32];
441
+ memset(buffer, 0, 32);
442
+ memcpy(buffer, src + i, len - i);
443
+ __m256i current_bytes = _mm256_loadu_si256((const __m256i *)(buffer));
444
+ previous = avxcheckUTF8Bytes(current_bytes, &previous, &has_error);
445
+ } else {
446
+ has_error = _mm256_or_si256(
447
+ _mm256_cmpgt_epi8(previous.carried_continuations,
448
+ _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
449
+ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
450
+ 9, 9, 9, 9, 9, 9, 9, 1)),
451
+ has_error);
452
+ }
453
+
454
+ return _mm256_testz_si256(has_error, has_error);
455
+ }
456
+
457
+ #endif // __AVX2__
458
+ #endif
@@ -0,0 +1,3 @@
1
+ module IsItUtf8
2
+ VERSION = '0.1.0'
3
+ end
metadata ADDED
@@ -0,0 +1,75 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: is_it_utf8
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - James Cook
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-01-24 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rake-compiler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1'
27
+ - !ruby/object:Gem::Dependency
28
+ name: minitest
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '5'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '5'
41
+ description:
42
+ email:
43
+ executables: []
44
+ extensions:
45
+ - ext/is_it_utf8/extconf.rb
46
+ extra_rdoc_files: []
47
+ files:
48
+ - ext/is_it_utf8/extconf.rb
49
+ - ext/is_it_utf8/is_it_utf8.c
50
+ - ext/is_it_utf8/simdutf8check.h
51
+ - lib/is_it_utf8/version.rb
52
+ homepage: https://github.com/jamescook/is_it_utf8
53
+ licenses:
54
+ - MIT
55
+ metadata: {}
56
+ post_install_message:
57
+ rdoc_options: []
58
+ require_paths:
59
+ - lib
60
+ required_ruby_version: !ruby/object:Gem::Requirement
61
+ requirements:
62
+ - - ">"
63
+ - !ruby/object:Gem::Version
64
+ version: '2.1'
65
+ required_rubygems_version: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ requirements: []
71
+ rubygems_version: 3.1.2
72
+ signing_key:
73
+ specification_version: 4
74
+ summary: Fast UTF8 Validator
75
+ test_files: []