@ohos-ports/bare-buffer 3.7.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +26 -0
- package/binding.c +1077 -0
- package/binding.gyp +32 -0
- package/binding.js +1 -0
- package/build/Release/bare_buffer.node +0 -0
- package/deps/libbase64/.clang-format +10 -0
- package/deps/libbase64/.gitattributes +1 -0
- package/deps/libbase64/.github/workflows/integrate.yml +45 -0
- package/deps/libbase64/.github/workflows/nightly.yml +28 -0
- package/deps/libbase64/.prettierrc +1 -0
- package/deps/libbase64/CMakeLists.txt +71 -0
- package/deps/libbase64/LICENSE +201 -0
- package/deps/libbase64/NOTICE +13 -0
- package/deps/libbase64/README.md +11 -0
- package/deps/libbase64/include/base64.h +323 -0
- package/deps/libbase64/package.json +25 -0
- package/deps/libbase64/src/base64.c +22 -0
- package/deps/libbase64/test/CMakeLists.txt +31 -0
- package/deps/libbase64/test/decode-invalid.c +35 -0
- package/deps/libbase64/test/decode-length.c +23 -0
- package/deps/libbase64/test/decode.c +36 -0
- package/deps/libbase64/test/encode-length.c +22 -0
- package/deps/libbase64/test/encode.c +42 -0
- package/deps/libbase64/test/fuzz/CMakeLists.txt +18 -0
- package/deps/libbase64/test/fuzz/decode/utf16le.c +53 -0
- package/deps/libbase64/test/fuzz/decode/utf8.c +47 -0
- package/deps/libbase64/test/fuzz/encode/utf16le.c +56 -0
- package/deps/libbase64/test/fuzz/encode/utf8.c +56 -0
- package/deps/libhex/.clang-format +10 -0
- package/deps/libhex/.gitattributes +1 -0
- package/deps/libhex/.github/workflows/integrate.yml +45 -0
- package/deps/libhex/.github/workflows/nightly.yml +28 -0
- package/deps/libhex/.prettierrc +1 -0
- package/deps/libhex/CMakeLists.txt +72 -0
- package/deps/libhex/LICENSE +201 -0
- package/deps/libhex/NOTICE +13 -0
- package/deps/libhex/README.md +11 -0
- package/deps/libhex/include/hex.h +163 -0
- package/deps/libhex/package.json +25 -0
- package/deps/libhex/src/hex.c +16 -0
- package/deps/libhex/test/CMakeLists.txt +31 -0
- package/deps/libhex/test/decode-invalid.c +37 -0
- package/deps/libhex/test/decode-length.c +23 -0
- package/deps/libhex/test/decode.c +25 -0
- package/deps/libhex/test/encode-length.c +22 -0
- package/deps/libhex/test/encode.c +25 -0
- package/deps/libhex/test/fuzz/CMakeLists.txt +18 -0
- package/deps/libhex/test/fuzz/decode/utf16le.c +53 -0
- package/deps/libhex/test/fuzz/decode/utf8.c +47 -0
- package/deps/libhex/test/fuzz/encode/utf16le.c +36 -0
- package/deps/libhex/test/fuzz/encode/utf8.c +36 -0
- package/deps/libutf/.clang-format +10 -0
- package/deps/libutf/.gitattributes +1 -0
- package/deps/libutf/.github/workflows/integrate.yml +45 -0
- package/deps/libutf/.github/workflows/nightly.yml +28 -0
- package/deps/libutf/.prettierrc +1 -0
- package/deps/libutf/CMakeLists.txt +105 -0
- package/deps/libutf/LICENSE +201 -0
- package/deps/libutf/NOTICE +13 -0
- package/deps/libutf/README.md +11 -0
- package/deps/libutf/include/utf/string.h +1655 -0
- package/deps/libutf/include/utf.h +132 -0
- package/deps/libutf/package.json +24 -0
- package/deps/libutf/src/ascii/validate.c +47 -0
- package/deps/libutf/src/endianness.c +19 -0
- package/deps/libutf/src/endianness.h +54 -0
- package/deps/libutf/src/latin1/convert-to-utf16.c +37 -0
- package/deps/libutf/src/latin1/convert-to-utf32.c +34 -0
- package/deps/libutf/src/latin1/convert-to-utf8.c +58 -0
- package/deps/libutf/src/latin1/length-from-utf16.c +26 -0
- package/deps/libutf/src/latin1/length-from-utf32.c +26 -0
- package/deps/libutf/src/latin1/length-from-utf8.c +34 -0
- package/deps/libutf/src/utf16/convert-to-latin1.c +41 -0
- package/deps/libutf/src/utf16/convert-to-utf32.c +56 -0
- package/deps/libutf/src/utf16/convert-to-utf8.c +80 -0
- package/deps/libutf/src/utf16/length-from-latin1.c +26 -0
- package/deps/libutf/src/utf16/length-from-utf32.c +33 -0
- package/deps/libutf/src/utf16/length-from-utf8.c +38 -0
- package/deps/libutf/src/utf16/validate.c +53 -0
- package/deps/libutf/src/utf32/convert-to-latin1.c +40 -0
- package/deps/libutf/src/utf32/convert-to-utf16.c +56 -0
- package/deps/libutf/src/utf32/convert-to-utf8.c +71 -0
- package/deps/libutf/src/utf32/length-from-latin1.c +26 -0
- package/deps/libutf/src/utf32/length-from-utf16.c +35 -0
- package/deps/libutf/src/utf32/length-from-utf8.c +35 -0
- package/deps/libutf/src/utf32/string.c +149 -0
- package/deps/libutf/src/utf32/validate.c +39 -0
- package/deps/libutf/src/utf8/convert-to-latin1.c +70 -0
- package/deps/libutf/src/utf8/convert-to-utf16.c +95 -0
- package/deps/libutf/src/utf8/convert-to-utf32.c +110 -0
- package/deps/libutf/src/utf8/length-from-latin1.c +32 -0
- package/deps/libutf/src/utf8/length-from-utf16.c +48 -0
- package/deps/libutf/src/utf8/length-from-utf32.c +35 -0
- package/deps/libutf/src/utf8/string.c +149 -0
- package/deps/libutf/src/utf8/validate.c +107 -0
- package/deps/libutf/test/CMakeLists.txt +56 -0
- package/deps/libutf/test/fuzz/CMakeLists.txt +26 -0
- package/deps/libutf/test/fuzz/corpus/.gitignore +2 -0
- package/deps/libutf/test/fuzz/crashes/.gitignore +2 -0
- package/deps/libutf/test/fuzz/latin1/convert-to-utf16.c +32 -0
- package/deps/libutf/test/fuzz/latin1/convert-to-utf32.c +32 -0
- package/deps/libutf/test/fuzz/latin1/convert-to-utf8.c +33 -0
- package/deps/libutf/test/fuzz/utf16/convert-to-latin1.c +36 -0
- package/deps/libutf/test/fuzz/utf16/convert-to-utf32.c +38 -0
- package/deps/libutf/test/fuzz/utf16/convert-to-utf8.c +38 -0
- package/deps/libutf/test/fuzz/utf32/convert-to-latin1.c +36 -0
- package/deps/libutf/test/fuzz/utf32/convert-to-utf16.c +38 -0
- package/deps/libutf/test/fuzz/utf32/convert-to-utf8.c +38 -0
- package/deps/libutf/test/fuzz/utf8/convert-to-latin1.c +33 -0
- package/deps/libutf/test/fuzz/utf8/convert-to-utf16.c +35 -0
- package/deps/libutf/test/fuzz/utf8/convert-to-utf32.c +35 -0
- package/deps/libutf/test/utf16/utf8-convert.c +45 -0
- package/deps/libutf/test/utf16/utf8-length.c +41 -0
- package/deps/libutf/test/utf16/validate-invalid.c +50 -0
- package/deps/libutf/test/utf16/validate.c +32 -0
- package/deps/libutf/test/utf32/string-append.c +74 -0
- package/deps/libutf/test/utf32/string-compare.c +108 -0
- package/deps/libutf/test/utf32/string-erase.c +79 -0
- package/deps/libutf/test/utf32/string-index.c +102 -0
- package/deps/libutf/test/utf32/string-insert.c +92 -0
- package/deps/libutf/test/utf32/string-last-index.c +71 -0
- package/deps/libutf/test/utf32/string-prepend.c +51 -0
- package/deps/libutf/test/utf32/string-replace.c +88 -0
- package/deps/libutf/test/utf32/string-reserve.c +99 -0
- package/deps/libutf/test/utf32/string-substring.c +93 -0
- package/deps/libutf/test/utf8/latin1-convert.c +23 -0
- package/deps/libutf/test/utf8/string-append.c +23 -0
- package/deps/libutf/test/utf8/string-compare.c +67 -0
- package/deps/libutf/test/utf8/string-erase.c +75 -0
- package/deps/libutf/test/utf8/string-index.c +99 -0
- package/deps/libutf/test/utf8/string-last-index.c +68 -0
- package/deps/libutf/test/utf8/string-prepend.c +23 -0
- package/deps/libutf/test/utf8/string-replace.c +74 -0
- package/deps/libutf/test/utf8/string-reserve.c +51 -0
- package/deps/libutf/test/utf8/string-substring.c +68 -0
- package/deps/libutf/test/utf8/utf16-convert.c +32 -0
- package/deps/libutf/test/utf8/utf16-length.c +29 -0
- package/deps/libutf/test/utf8/validate-invalid.c +26 -0
- package/deps/libutf/test/utf8/validate.c +21 -0
- package/global.d.ts +21 -0
- package/global.js +6 -0
- package/index.d.ts +769 -0
- package/index.js +1339 -0
- package/lib/ascii.js +36 -0
- package/lib/base64.js +25 -0
- package/lib/base64url.js +25 -0
- package/lib/checked.js +23 -0
- package/lib/constants.d.ts +7 -0
- package/lib/constants.js +3 -0
- package/lib/hex.js +20 -0
- package/lib/latin1.js +14 -0
- package/lib/utf16le.js +20 -0
- package/lib/utf8.js +18 -0
- package/package.json +64 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
|
|
3
|
+
#include "../../include/utf.h"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
7
|
+
*
|
|
8
|
+
* Copyright 2020 The simdutf authors
|
|
9
|
+
*
|
|
10
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
11
|
+
* you may not use this file except in compliance with the License.
|
|
12
|
+
* You may obtain a copy of the License at
|
|
13
|
+
*
|
|
14
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
15
|
+
*
|
|
16
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
+
* See the License for the specific language governing permissions and
|
|
20
|
+
* limitations under the License.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
size_t
|
|
24
|
+
utf16_length_from_latin1(const utf8_t *data, size_t len) {
|
|
25
|
+
return len;
|
|
26
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
|
|
3
|
+
#include "../../include/utf.h"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
7
|
+
*
|
|
8
|
+
* Copyright 2020 The simdutf authors
|
|
9
|
+
*
|
|
10
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
11
|
+
* you may not use this file except in compliance with the License.
|
|
12
|
+
* You may obtain a copy of the License at
|
|
13
|
+
*
|
|
14
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
15
|
+
*
|
|
16
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
+
* See the License for the specific language governing permissions and
|
|
20
|
+
* limitations under the License.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
size_t
|
|
24
|
+
utf16_length_from_utf32(const utf32_t *data, size_t len) {
|
|
25
|
+
size_t counter = 0;
|
|
26
|
+
|
|
27
|
+
for (size_t i = 0; i < len; i++) {
|
|
28
|
+
counter++;
|
|
29
|
+
counter += data[i] > 0xffff;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return counter;
|
|
33
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
|
|
4
|
+
#include "../../include/utf.h"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
8
|
+
*
|
|
9
|
+
* Copyright 2020 The simdutf authors
|
|
10
|
+
*
|
|
11
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
12
|
+
* you may not use this file except in compliance with the License.
|
|
13
|
+
* You may obtain a copy of the License at
|
|
14
|
+
*
|
|
15
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
16
|
+
*
|
|
17
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
18
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
19
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
20
|
+
* See the License for the specific language governing permissions and
|
|
21
|
+
* limitations under the License.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
size_t
|
|
25
|
+
utf16_length_from_utf8(const utf8_t *data, size_t len) {
|
|
26
|
+
size_t counter = 0;
|
|
27
|
+
|
|
28
|
+
for (size_t i = 0; i < len; i++) {
|
|
29
|
+
if ((int8_t) data[i] > -65) {
|
|
30
|
+
counter++;
|
|
31
|
+
}
|
|
32
|
+
if (data[i] >= 240) {
|
|
33
|
+
counter++;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return counter;
|
|
38
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#include <stdbool.h>
|
|
2
|
+
#include <stddef.h>
|
|
3
|
+
#include <stdint.h>
|
|
4
|
+
|
|
5
|
+
#include "../../include/utf.h"
|
|
6
|
+
#include "../endianness.h"
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
10
|
+
*
|
|
11
|
+
* Copyright 2020 The simdutf authors
|
|
12
|
+
*
|
|
13
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
14
|
+
* you may not use this file except in compliance with the License.
|
|
15
|
+
* You may obtain a copy of the License at
|
|
16
|
+
*
|
|
17
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
18
|
+
*
|
|
19
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
20
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
21
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
22
|
+
* See the License for the specific language governing permissions and
|
|
23
|
+
* limitations under the License.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
bool
|
|
27
|
+
utf16le_validate(const utf16_t *data, size_t len) {
|
|
28
|
+
uint64_t pos = 0;
|
|
29
|
+
uint16_t word, diff;
|
|
30
|
+
|
|
31
|
+
while (pos < len) {
|
|
32
|
+
word = utf_is_be() ? utf_swap_uint16(data[pos]) : data[pos];
|
|
33
|
+
if ((word & 0xf800) == 0xd800) {
|
|
34
|
+
if (pos + 1 >= len) {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
diff = word - 0xd800;
|
|
38
|
+
if (diff > 0x3ff) {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
word = utf_is_be() ? utf_swap_uint16(data[pos + 1]) : data[pos + 1];
|
|
42
|
+
diff = word - 0xdc00;
|
|
43
|
+
if (diff > 0x3ff) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
pos += 2;
|
|
47
|
+
} else {
|
|
48
|
+
pos++;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
|
|
4
|
+
#include "../../include/utf.h"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
8
|
+
*
|
|
9
|
+
* Copyright 2020 The simdutf authors
|
|
10
|
+
*
|
|
11
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
12
|
+
* you may not use this file except in compliance with the License.
|
|
13
|
+
* You may obtain a copy of the License at
|
|
14
|
+
*
|
|
15
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
16
|
+
*
|
|
17
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
18
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
19
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
20
|
+
* See the License for the specific language governing permissions and
|
|
21
|
+
* limitations under the License.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
size_t
|
|
25
|
+
utf32_convert_to_latin1(const utf32_t *data, size_t len, latin1_t *result) {
|
|
26
|
+
size_t pos = 0;
|
|
27
|
+
uint32_t word, overflow = 0;
|
|
28
|
+
latin1_t *start = result;
|
|
29
|
+
|
|
30
|
+
while (pos < len) {
|
|
31
|
+
word = data[pos];
|
|
32
|
+
overflow |= word;
|
|
33
|
+
*result++ = word & 0xff;
|
|
34
|
+
pos++;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
if (overflow & 0xffffff00) return 0;
|
|
38
|
+
|
|
39
|
+
return (size_t) (result - start);
|
|
40
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
|
|
4
|
+
#include "../../include/utf.h"
|
|
5
|
+
#include "../endianness.h"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
9
|
+
*
|
|
10
|
+
* Copyright 2020 The simdutf authors
|
|
11
|
+
*
|
|
12
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
* you may not use this file except in compliance with the License.
|
|
14
|
+
* You may obtain a copy of the License at
|
|
15
|
+
*
|
|
16
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
*
|
|
18
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
* See the License for the specific language governing permissions and
|
|
22
|
+
* limitations under the License.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
size_t
|
|
26
|
+
utf32_convert_to_utf16le(const utf32_t *data, size_t len, utf16_t *result) {
|
|
27
|
+
size_t pos = 0;
|
|
28
|
+
uint32_t word;
|
|
29
|
+
utf16_t *start = result;
|
|
30
|
+
|
|
31
|
+
while (pos < len) {
|
|
32
|
+
word = data[pos];
|
|
33
|
+
if ((word & 0xffff0000) == 0) {
|
|
34
|
+
if (word >= 0xd800 && word <= 0xdfff) {
|
|
35
|
+
return 0;
|
|
36
|
+
}
|
|
37
|
+
*result++ = utf_is_be() ? utf_swap_uint16((uint16_t) word) : (uint16_t) word;
|
|
38
|
+
} else {
|
|
39
|
+
if (word > 0x10ffff) {
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
word -= 0x10000;
|
|
43
|
+
uint16_t high_surrogate = (uint16_t) (0xd800 + (word >> 10));
|
|
44
|
+
uint16_t low_surrogate = (uint16_t) (0xdc00 + (word & 0x3ff));
|
|
45
|
+
if (utf_is_be()) {
|
|
46
|
+
high_surrogate = utf_swap_uint16(high_surrogate);
|
|
47
|
+
low_surrogate = utf_swap_uint16(low_surrogate);
|
|
48
|
+
}
|
|
49
|
+
*result++ = high_surrogate;
|
|
50
|
+
*result++ = low_surrogate;
|
|
51
|
+
}
|
|
52
|
+
pos++;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return (size_t) (result - start);
|
|
56
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
#include <string.h>
|
|
4
|
+
|
|
5
|
+
#include "../../include/utf.h"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
9
|
+
*
|
|
10
|
+
* Copyright 2020 The simdutf authors
|
|
11
|
+
*
|
|
12
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
* you may not use this file except in compliance with the License.
|
|
14
|
+
* You may obtain a copy of the License at
|
|
15
|
+
*
|
|
16
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
*
|
|
18
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
* See the License for the specific language governing permissions and
|
|
22
|
+
* limitations under the License.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
size_t
|
|
26
|
+
utf32_convert_to_utf8(const utf32_t *data, size_t len, utf8_t *result) {
|
|
27
|
+
size_t pos = 0;
|
|
28
|
+
uint32_t word;
|
|
29
|
+
utf8_t *start = result;
|
|
30
|
+
|
|
31
|
+
while (pos < len) {
|
|
32
|
+
if (pos + 2 <= len) {
|
|
33
|
+
uint64_t v;
|
|
34
|
+
memcpy(&v, data + pos, sizeof(v));
|
|
35
|
+
if ((v & 0xffffff80ffffff80) == 0) {
|
|
36
|
+
*result++ = (uint8_t) data[pos];
|
|
37
|
+
*result++ = (uint8_t) data[pos + 1];
|
|
38
|
+
pos += 2;
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
word = data[pos];
|
|
43
|
+
if ((word & 0xffffff80) == 0) {
|
|
44
|
+
*result++ = (uint8_t) word;
|
|
45
|
+
pos++;
|
|
46
|
+
} else if ((word & 0xfffff800) == 0) {
|
|
47
|
+
*result++ = (uint8_t) (word >> 6) | 0b11000000;
|
|
48
|
+
*result++ = (word & 0b111111) | 0b10000000;
|
|
49
|
+
pos++;
|
|
50
|
+
} else if ((word & 0xffff0000) == 0) {
|
|
51
|
+
if (word >= 0xd800 && word <= 0xdfff) {
|
|
52
|
+
return 0;
|
|
53
|
+
}
|
|
54
|
+
*result++ = (uint8_t) (word >> 12) | 0b11100000;
|
|
55
|
+
*result++ = ((word >> 6) & 0b111111) | 0b10000000;
|
|
56
|
+
*result++ = (word & 0b111111) | 0b10000000;
|
|
57
|
+
pos++;
|
|
58
|
+
} else {
|
|
59
|
+
if (word > 0x10ffff) {
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
*result++ = (uint8_t) (word >> 18) | 0b11110000;
|
|
63
|
+
*result++ = ((word >> 12) & 0b111111) | 0b10000000;
|
|
64
|
+
*result++ = ((word >> 6) & 0b111111) | 0b10000000;
|
|
65
|
+
*result++ = (word & 0b111111) | 0b10000000;
|
|
66
|
+
pos++;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return (size_t) (result - start);
|
|
71
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
|
|
3
|
+
#include "../../include/utf.h"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
7
|
+
*
|
|
8
|
+
* Copyright 2020 The simdutf authors
|
|
9
|
+
*
|
|
10
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
11
|
+
* you may not use this file except in compliance with the License.
|
|
12
|
+
* You may obtain a copy of the License at
|
|
13
|
+
*
|
|
14
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
15
|
+
*
|
|
16
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
+
* See the License for the specific language governing permissions and
|
|
20
|
+
* limitations under the License.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
size_t
|
|
24
|
+
utf32_length_from_latin1(const utf8_t *data, size_t len) {
|
|
25
|
+
return len;
|
|
26
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
|
|
4
|
+
#include "../../include/utf.h"
|
|
5
|
+
#include "../endianness.h"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
9
|
+
*
|
|
10
|
+
* Copyright 2020 The simdutf authors
|
|
11
|
+
*
|
|
12
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
* you may not use this file except in compliance with the License.
|
|
14
|
+
* You may obtain a copy of the License at
|
|
15
|
+
*
|
|
16
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
*
|
|
18
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
* See the License for the specific language governing permissions and
|
|
22
|
+
* limitations under the License.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
size_t
|
|
26
|
+
utf32_length_from_utf16le(const utf16_t *data, size_t len) {
|
|
27
|
+
size_t counter = 0;
|
|
28
|
+
|
|
29
|
+
for (size_t i = 0; i < len; i++) {
|
|
30
|
+
uint16_t word = utf_is_be() ? utf_swap_uint16(data[i]) : data[i];
|
|
31
|
+
counter += ((word & 0xfc00) != 0xdc00);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return counter;
|
|
35
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
|
|
4
|
+
#include "../../include/utf.h"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
8
|
+
*
|
|
9
|
+
* Copyright 2020 The simdutf authors
|
|
10
|
+
*
|
|
11
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
12
|
+
* you may not use this file except in compliance with the License.
|
|
13
|
+
* You may obtain a copy of the License at
|
|
14
|
+
*
|
|
15
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
16
|
+
*
|
|
17
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
18
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
19
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
20
|
+
* See the License for the specific language governing permissions and
|
|
21
|
+
* limitations under the License.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
size_t
|
|
25
|
+
utf32_length_from_utf8(const utf8_t *data, size_t len) {
|
|
26
|
+
size_t counter = 0;
|
|
27
|
+
|
|
28
|
+
for (size_t i = 0; i < len; i++) {
|
|
29
|
+
if ((int8_t) data[i] > -65) {
|
|
30
|
+
counter++;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return counter;
|
|
35
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
#include <stdbool.h>
|
|
2
|
+
#include <stddef.h>
|
|
3
|
+
|
|
4
|
+
#include "../../include/utf.h"
|
|
5
|
+
#include "../../include/utf/string.h"
|
|
6
|
+
|
|
7
|
+
extern void
|
|
8
|
+
utf32_string_init(utf32_string_t *string);
|
|
9
|
+
|
|
10
|
+
extern utf32_string_view_t
|
|
11
|
+
utf32_string_view_init(const utf32_t *data, size_t len);
|
|
12
|
+
|
|
13
|
+
extern void
|
|
14
|
+
utf32_string_destroy(utf32_string_t *string);
|
|
15
|
+
|
|
16
|
+
extern int
|
|
17
|
+
utf32_string_reserve(utf32_string_t *string, size_t len);
|
|
18
|
+
|
|
19
|
+
extern int
|
|
20
|
+
utf32_string_shrink_to_fit(utf32_string_t *string);
|
|
21
|
+
|
|
22
|
+
extern utf32_string_view_t
|
|
23
|
+
utf32_string_view(const utf32_string_t *string);
|
|
24
|
+
|
|
25
|
+
extern void
|
|
26
|
+
utf32_string_clear(utf32_string_t *string);
|
|
27
|
+
|
|
28
|
+
extern bool
|
|
29
|
+
utf32_string_empty(const utf32_string_t *string);
|
|
30
|
+
|
|
31
|
+
extern bool
|
|
32
|
+
utf32_string_view_empty(const utf32_string_view_t view);
|
|
33
|
+
|
|
34
|
+
extern int
|
|
35
|
+
utf32_string_copy(const utf32_string_t *string, utf32_string_t *result);
|
|
36
|
+
|
|
37
|
+
extern int
|
|
38
|
+
utf32_string_view_copy(const utf32_string_view_t view, utf32_string_t *result);
|
|
39
|
+
|
|
40
|
+
extern int
|
|
41
|
+
utf32_string_append(utf32_string_t *string, const utf32_string_t *other);
|
|
42
|
+
|
|
43
|
+
extern int
|
|
44
|
+
utf32_string_append_view(utf32_string_t *string, const utf32_string_view_t view);
|
|
45
|
+
|
|
46
|
+
extern int
|
|
47
|
+
utf32_string_append_character(utf32_string_t *string, utf32_t c);
|
|
48
|
+
|
|
49
|
+
extern int
|
|
50
|
+
utf32_string_append_literal(utf32_string_t *string, const utf32_t *literal, size_t n);
|
|
51
|
+
|
|
52
|
+
extern int
|
|
53
|
+
utf32_string_prepend(utf32_string_t *string, const utf32_string_t *other);
|
|
54
|
+
|
|
55
|
+
extern int
|
|
56
|
+
utf32_string_prepend_view(utf32_string_t *string, const utf32_string_view_t view);
|
|
57
|
+
|
|
58
|
+
extern int
|
|
59
|
+
utf32_string_prepend_character(utf32_string_t *string, utf32_t c);
|
|
60
|
+
|
|
61
|
+
extern int
|
|
62
|
+
utf32_string_prepend_literal(utf32_string_t *string, const utf32_t *literal, size_t n);
|
|
63
|
+
|
|
64
|
+
extern int
|
|
65
|
+
utf32_string_insert(utf32_string_t *string, size_t pos, const utf32_string_t *other);
|
|
66
|
+
|
|
67
|
+
extern int
|
|
68
|
+
utf32_string_insert_view(utf32_string_t *string, size_t pos, const utf32_string_view_t other);
|
|
69
|
+
|
|
70
|
+
extern int
|
|
71
|
+
utf32_string_insert_character(utf32_string_t *string, size_t pos, utf32_t c);
|
|
72
|
+
|
|
73
|
+
extern int
|
|
74
|
+
utf32_string_insert_literal(utf32_string_t *string, size_t pos, const utf32_t *literal, size_t n);
|
|
75
|
+
|
|
76
|
+
extern int
|
|
77
|
+
utf32_string_replace(utf32_string_t *string, size_t pos, size_t len, const utf32_string_t *replacement);
|
|
78
|
+
|
|
79
|
+
extern int
|
|
80
|
+
utf32_string_replace_view(utf32_string_t *string, size_t pos, size_t len, const utf32_string_view_t replacement);
|
|
81
|
+
|
|
82
|
+
extern int
|
|
83
|
+
utf32_string_replace_character(utf32_string_t *string, size_t pos, size_t len, utf32_t c);
|
|
84
|
+
|
|
85
|
+
extern int
|
|
86
|
+
utf32_string_replace_literal(utf32_string_t *string, size_t pos, size_t len, const utf32_t *literal, size_t n);
|
|
87
|
+
|
|
88
|
+
extern int
|
|
89
|
+
utf32_string_erase(utf32_string_t *string, size_t pos, size_t len);
|
|
90
|
+
|
|
91
|
+
extern int
|
|
92
|
+
utf32_string_concat(const utf32_string_t *string, const utf32_string_t *other, utf32_string_t *result);
|
|
93
|
+
|
|
94
|
+
extern int
|
|
95
|
+
utf32_string_view_concat(const utf32_string_view_t view, const utf32_string_t *other, utf32_string_t *result);
|
|
96
|
+
|
|
97
|
+
extern int
|
|
98
|
+
utf32_string_concat_view(const utf32_string_t *string, const utf32_string_view_t other, utf32_string_t *result);
|
|
99
|
+
|
|
100
|
+
extern int
|
|
101
|
+
utf32_string_view_concat_view(const utf32_string_view_t view, const utf32_string_view_t other, utf32_string_t *result);
|
|
102
|
+
|
|
103
|
+
extern int
|
|
104
|
+
utf32_string_concat_character(const utf32_string_t *string, utf32_t c, utf32_string_t *result);
|
|
105
|
+
|
|
106
|
+
extern int
|
|
107
|
+
utf32_string_view_concat_character(const utf32_string_view_t view, utf32_t c, utf32_string_t *result);
|
|
108
|
+
|
|
109
|
+
extern int
|
|
110
|
+
utf32_string_concat_literal(const utf32_string_t *string, const utf32_t *literal, size_t n, utf32_string_t *result);
|
|
111
|
+
|
|
112
|
+
extern int
|
|
113
|
+
utf32_string_view_concat_literal(const utf32_string_view_t view, const utf32_t *literal, size_t n, utf32_string_t *result);
|
|
114
|
+
|
|
115
|
+
extern int
|
|
116
|
+
utf32_string_compare(const utf32_string_t *string, const utf32_string_t *other);
|
|
117
|
+
|
|
118
|
+
extern int
|
|
119
|
+
utf32_string_view_compare(const utf32_string_view_t view, const utf32_string_view_t other);
|
|
120
|
+
|
|
121
|
+
extern int
|
|
122
|
+
utf32_string_compare_literal(const utf32_string_t *string, const utf32_t *literal, size_t n);
|
|
123
|
+
|
|
124
|
+
extern int
|
|
125
|
+
utf32_string_view_compare_literal(const utf32_string_view_t view, const utf32_t *literal, size_t n);
|
|
126
|
+
|
|
127
|
+
extern utf32_string_view_t
|
|
128
|
+
utf32_string_substring(const utf32_string_t *string, size_t start, size_t end);
|
|
129
|
+
|
|
130
|
+
extern utf32_string_view_t
|
|
131
|
+
utf32_string_view_substring(const utf32_string_view_t view, size_t start, size_t end);
|
|
132
|
+
|
|
133
|
+
extern int
|
|
134
|
+
utf32_string_substring_copy(const utf32_string_t *string, size_t start, size_t end, utf32_string_t *result);
|
|
135
|
+
|
|
136
|
+
extern int
|
|
137
|
+
utf32_string_view_substring_copy(const utf32_string_view_t view, size_t start, size_t end, utf32_string_t *result);
|
|
138
|
+
|
|
139
|
+
extern size_t
|
|
140
|
+
utf32_string_index_of_character(const utf32_string_t *string, size_t pos, utf32_t c);
|
|
141
|
+
|
|
142
|
+
extern size_t
|
|
143
|
+
utf32_string_view_index_of_character(const utf32_string_view_t view, size_t pos, utf32_t c);
|
|
144
|
+
|
|
145
|
+
extern size_t
|
|
146
|
+
utf32_string_last_index_of_character(const utf32_string_t *string, size_t pos, utf32_t c);
|
|
147
|
+
|
|
148
|
+
extern size_t
|
|
149
|
+
utf32_string_view_last_index_of_character(const utf32_string_view_t view, size_t pos, utf32_t c);
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#include <stdbool.h>
|
|
2
|
+
#include <stddef.h>
|
|
3
|
+
#include <stdint.h>
|
|
4
|
+
|
|
5
|
+
#include "../../include/utf.h"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
9
|
+
*
|
|
10
|
+
* Copyright 2020 The simdutf authors
|
|
11
|
+
*
|
|
12
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
* you may not use this file except in compliance with the License.
|
|
14
|
+
* You may obtain a copy of the License at
|
|
15
|
+
*
|
|
16
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
*
|
|
18
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
* See the License for the specific language governing permissions and
|
|
22
|
+
* limitations under the License.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
bool
|
|
26
|
+
utf32_validate(const utf32_t *data, size_t len) {
|
|
27
|
+
uint64_t pos = 0;
|
|
28
|
+
uint32_t word;
|
|
29
|
+
|
|
30
|
+
while (pos < len) {
|
|
31
|
+
word = data[pos];
|
|
32
|
+
if (word > 0x10ffff || (word >= 0xd800 && word <= 0xdfff)) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
pos++;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#include <stddef.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
#include <string.h>
|
|
4
|
+
|
|
5
|
+
#include "../../include/utf.h"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Modified from https://github.com/simdutf/simdutf
|
|
9
|
+
*
|
|
10
|
+
* Copyright 2020 The simdutf authors
|
|
11
|
+
*
|
|
12
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
13
|
+
* you may not use this file except in compliance with the License.
|
|
14
|
+
* You may obtain a copy of the License at
|
|
15
|
+
*
|
|
16
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
17
|
+
*
|
|
18
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
19
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
20
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
21
|
+
* See the License for the specific language governing permissions and
|
|
22
|
+
* limitations under the License.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
size_t
|
|
26
|
+
utf8_convert_to_latin1(const utf8_t *data, size_t len, latin1_t *result) {
|
|
27
|
+
size_t pos = 0;
|
|
28
|
+
latin1_t *start = result;
|
|
29
|
+
|
|
30
|
+
while (pos < len) {
|
|
31
|
+
if (pos + 16 <= len) {
|
|
32
|
+
uint64_t v1;
|
|
33
|
+
memcpy(&v1, data + pos, sizeof(uint64_t));
|
|
34
|
+
uint64_t v2;
|
|
35
|
+
memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
|
|
36
|
+
uint64_t v = v1 | v2;
|
|
37
|
+
if ((v & 0x8080808080808080) == 0) {
|
|
38
|
+
size_t final_pos = pos + 16;
|
|
39
|
+
while (pos < final_pos) {
|
|
40
|
+
*result++ = data[pos];
|
|
41
|
+
pos++;
|
|
42
|
+
}
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
uint8_t leading_byte = data[pos];
|
|
47
|
+
if (leading_byte < 0b10000000) {
|
|
48
|
+
*result++ = leading_byte;
|
|
49
|
+
pos++;
|
|
50
|
+
} else if ((leading_byte & 0b11100000) == 0b11000000) {
|
|
51
|
+
if (pos + 1 >= len) {
|
|
52
|
+
return 0;
|
|
53
|
+
}
|
|
54
|
+
if ((data[pos + 1] & 0b11000000) != 0b10000000) {
|
|
55
|
+
return 0;
|
|
56
|
+
}
|
|
57
|
+
uint32_t code_point = (uint32_t) (((leading_byte & 0b00011111) << 6) |
|
|
58
|
+
(data[pos + 1] & 0b00111111));
|
|
59
|
+
if (code_point < 0x80 || code_point > 0xff) {
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
*result++ = (uint8_t) code_point;
|
|
63
|
+
pos += 2;
|
|
64
|
+
} else {
|
|
65
|
+
return 0;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return (size_t) (result - start);
|
|
70
|
+
}
|