mol_vary 0.0.94 → 0.0.96

package/node.mjs

@@ -77,54 +77,57 @@ var $;
 "use strict";
 var $;
 (function ($) {
-    function $mol_charset_encode(str) {
+    function $mol_charset_ucf_encode(str) {
         const buf = $mol_charset_buffer(str.length * 3);
-        return buf.slice(0, $mol_charset_encode_to(str, buf));
-    }
-    $.$mol_charset_encode = $mol_charset_encode;
-    function $mol_charset_encode_to(str, buf, from = 0) {
+        return buf.slice(0, $mol_charset_ucf_encode_to(str, buf));
+    }
+    $.$mol_charset_ucf_encode = $mol_charset_ucf_encode;
+    const fast_char = `0123456789.,:;()?!-'" \n`;
+    const fast_map = new Array(0x80).fill(0);
+    for (let i = 0; i < fast_char.length; ++i)
+        fast_map[fast_char[i].charCodeAt(0)] = i | 0x80;
+    function $mol_charset_ucf_encode_to(str, buf, from = 0) {
         let pos = from;
+        let mode = 0x9C;
         for (let i = 0; i < str.length; i++) {
             let code = str.charCodeAt(i);
+            if (code >= 0xd800 && code < 0xe000)
+                code = ((code - 0xd800) << 10) + str.charCodeAt(++i) + 0x2400;
             if (code < 0x80) {
+                if (mode !== 0x9C) {
+                    const fast = fast_map[code];
+                    if (fast)
+                        code = fast;
+                    else
+                        buf[pos++] = mode = 0x9C;
+                }
                 buf[pos++] = code;
             }
-            else if (code < 0x800) {
-                buf[pos++] = 0xc0 | (code >> 6);
-                buf[pos++] = 0x80 | (code & 0x3f);
+            else if (code < 0x32_00) {
+                const page = (code >> 7) + 0x9C;
+                if (mode !== page)
+                    buf[pos++] = mode = page;
+                buf[pos++] = code & 0x7F;
             }
-            else if (code < 0xd800 || code >= 0xe000) {
-                buf[pos++] = 0xe0 | (code >> 12);
-                buf[pos++] = 0x80 | ((code >> 6) & 0x3f);
-                buf[pos++] = 0x80 | (code & 0x3f);
+            else if (code < 0x04_20_00) {
+                code -= 0x2000;
+                const page = (code >> 15) + 0x98;
+                if (mode !== page)
+                    buf[pos++] = mode = page;
+                buf[pos++] = code & 0x7F;
+                buf[pos++] = code >> 7;
             }
             else {
-                const point = ((code - 0xd800) << 10) + str.charCodeAt(++i) + 0x2400;
-                buf[pos++] = 0xf0 | (point >> 18);
-                buf[pos++] = 0x80 | ((point >> 12) & 0x3f);
-                buf[pos++] = 0x80 | ((point >> 6) & 0x3f);
-                buf[pos++] = 0x80 | (point & 0x3f);
+                if (mode !== 0x97)
+                    buf[pos++] = mode = 0x97;
+                buf[pos++] = code & 0x7F;
+                buf[pos++] = code >> 7;
+                buf[pos++] = code >> 15;
             }
         }
         return pos - from;
     }
-    $.$mol_charset_encode_to = $mol_charset_encode_to;
-    function $mol_charset_encode_size(str) {
-        let size = 0;
-        for (let i = 0; i < str.length; i++) {
-            let code = str.charCodeAt(i);
-            if (code < 0x80)
-                size += 1;
-            else if (code < 0x800)
-                size += 2;
-            else if (code < 0xd800 || code >= 0xe000)
-                size += 3;
-            else
-                size += 4;
-        }
-        return size;
-    }
-    $.$mol_charset_encode_size = $mol_charset_encode_size;
+    $.$mol_charset_ucf_encode_to = $mol_charset_ucf_encode_to;
 })($ || ($ = {}));
 
 ;
@@ -153,21 +156,43 @@ var $;
     $.$mol_bigint_decode = $mol_bigint_decode;
 })($ || ($ = {}));
 
-;
-"use strict";
-
 ;
 "use strict";
 var $;
 (function ($) {
-    const decoders = {};
-    function $mol_charset_decode(buffer, encoding = 'utf8') {
-        let decoder = decoders[encoding];
-        if (!decoder)
-            decoder = decoders[encoding] = new TextDecoder(encoding);
-        return decoder.decode(buffer);
-    }
-    $.$mol_charset_decode = $mol_charset_decode;
+    const fast_char = `0123456789.,:;()?!-'" \n`;
+    function $mol_charset_ucf_decode(buffer, mode = 0x9C) {
+        let text = '';
+        let pos = 0;
+        let page_offset = 0;
+        while (pos < buffer.length) {
+            let code = buffer[pos++];
+            if (code < 0x80) {
+                if (mode < 0x9C)
+                    code |= buffer[pos++] << 7;
+                if (mode === 0x97)
+                    code |= buffer[pos++] << 15;
+                text += String.fromCodePoint(page_offset + code);
+            }
+            else if (code < 0x97) {
+                text += fast_char[code - 0x80];
+            }
+            else if (code >= 0x9C) {
+                mode = code;
+                page_offset = (mode - 0x9C) << 7;
+            }
+            else if (code === 0x97) {
+                mode = code;
+                page_offset = 0;
+            }
+            else {
+                mode = code;
+                page_offset = ((mode - 0x98) << 15) + 0x20_00;
+            }
+        }
+        return text;
+    }
+    $.$mol_charset_ucf_decode = $mol_charset_ucf_decode;
 })($ || ($ = {}));
 
 ;
@@ -2474,7 +2499,7 @@ var $;
                 const len_max = val.length * 3;
                 const len_size = calc_size(len_max);
                 acquire(len_max);
-                const len = $mol_charset_encode_to(val, this.array, pos + len_size);
+                const len = $mol_charset_ucf_encode_to(val, this.array, pos + len_size);
                 dump_unum($mol_vary_tip.text, len, len_max);
                 pos += len;
                 release(len_max - len);
@@ -2683,7 +2708,7 @@ var $;
             };
             const read_text = (kind) => {
                 const len = read_unum(kind);
-                const text = $mol_charset_decode(new Uint8Array(array.buffer, array.byteOffset + pos, len));
+                const text = $mol_charset_ucf_decode(new Uint8Array(array.buffer, array.byteOffset + pos, len));
                 pos += len;
                 stream.push(text);
                 return text;

package/node.test.js

@@ -68,54 +68,57 @@ var $;
 "use strict";
 var $;
 (function ($) {
-    function $mol_charset_encode(str) {
+    function $mol_charset_ucf_encode(str) {
         const buf = $mol_charset_buffer(str.length * 3);
-        return buf.slice(0, $mol_charset_encode_to(str, buf));
+        return buf.slice(0, $mol_charset_ucf_encode_to(str, buf));
     }
-    $.$mol_charset_encode = $mol_charset_encode;
-    function $mol_charset_encode_to(str, buf, from = 0) {
+    $.$mol_charset_ucf_encode = $mol_charset_ucf_encode;
+    const fast_char = `0123456789.,:;()?!-'" \n`;
+    const fast_map = new Array(0x80).fill(0);
+    for (let i = 0; i < fast_char.length; ++i)
+        fast_map[fast_char[i].charCodeAt(0)] = i | 0x80;
+    function $mol_charset_ucf_encode_to(str, buf, from = 0) {
         let pos = from;
+        let mode = 0x9C;
         for (let i = 0; i < str.length; i++) {
             let code = str.charCodeAt(i);
+            if (code >= 0xd800 && code < 0xe000)
+                code = ((code - 0xd800) << 10) + str.charCodeAt(++i) + 0x2400;
             if (code < 0x80) {
+                if (mode !== 0x9C) {
+                    const fast = fast_map[code];
+                    if (fast)
+                        code = fast;
+                    else
+                        buf[pos++] = mode = 0x9C;
+                }
                 buf[pos++] = code;
             }
-            else if (code < 0x800) {
-                buf[pos++] = 0xc0 | (code >> 6);
-                buf[pos++] = 0x80 | (code & 0x3f);
+            else if (code < 0x32_00) {
+                const page = (code >> 7) + 0x9C;
+                if (mode !== page)
+                    buf[pos++] = mode = page;
+                buf[pos++] = code & 0x7F;
             }
-            else if (code < 0xd800 || code >= 0xe000) {
-                buf[pos++] = 0xe0 | (code >> 12);
-                buf[pos++] = 0x80 | ((code >> 6) & 0x3f);
-                buf[pos++] = 0x80 | (code & 0x3f);
+            else if (code < 0x04_20_00) {
+                code -= 0x2000;
+                const page = (code >> 15) + 0x98;
+                if (mode !== page)
+                    buf[pos++] = mode = page;
+                buf[pos++] = code & 0x7F;
+                buf[pos++] = code >> 7;
             }
             else {
-                const point = ((code - 0xd800) << 10) + str.charCodeAt(++i) + 0x2400;
-                buf[pos++] = 0xf0 | (point >> 18);
-                buf[pos++] = 0x80 | ((point >> 12) & 0x3f);
-                buf[pos++] = 0x80 | ((point >> 6) & 0x3f);
-                buf[pos++] = 0x80 | (point & 0x3f);
+                if (mode !== 0x97)
+                    buf[pos++] = mode = 0x97;
+                buf[pos++] = code & 0x7F;
+                buf[pos++] = code >> 7;
+                buf[pos++] = code >> 15;
             }
         }
         return pos - from;
     }
-    $.$mol_charset_encode_to = $mol_charset_encode_to;
-    function $mol_charset_encode_size(str) {
-        let size = 0;
-        for (let i = 0; i < str.length; i++) {
-            let code = str.charCodeAt(i);
-            if (code < 0x80)
-                size += 1;
-            else if (code < 0x800)
-                size += 2;
-            else if (code < 0xd800 || code >= 0xe000)
-                size += 3;
-            else
-                size += 4;
-        }
-        return size;
-    }
-    $.$mol_charset_encode_size = $mol_charset_encode_size;
+    $.$mol_charset_ucf_encode_to = $mol_charset_ucf_encode_to;
 })($ || ($ = {}));
 
 ;
@@ -144,21 +147,43 @@ var $;
     $.$mol_bigint_decode = $mol_bigint_decode;
 })($ || ($ = {}));
 
-;
-"use strict";
-
 ;
 "use strict";
 var $;
 (function ($) {
-    const decoders = {};
-    function $mol_charset_decode(buffer, encoding = 'utf8') {
-        let decoder = decoders[encoding];
-        if (!decoder)
-            decoder = decoders[encoding] = new TextDecoder(encoding);
-        return decoder.decode(buffer);
+    const fast_char = `0123456789.,:;()?!-'" \n`;
+    function $mol_charset_ucf_decode(buffer, mode = 0x9C) {
+        let text = '';
+        let pos = 0;
+        let page_offset = 0;
+        while (pos < buffer.length) {
+            let code = buffer[pos++];
+            if (code < 0x80) {
+                if (mode < 0x9C)
+                    code |= buffer[pos++] << 7;
+                if (mode === 0x97)
+                    code |= buffer[pos++] << 15;
+                text += String.fromCodePoint(page_offset + code);
+            }
+            else if (code < 0x97) {
+                text += fast_char[code - 0x80];
+            }
+            else if (code >= 0x9C) {
+                mode = code;
+                page_offset = (mode - 0x9C) << 7;
+            }
+            else if (code === 0x97) {
+                mode = code;
+                page_offset = 0;
+            }
+            else {
+                mode = code;
+                page_offset = ((mode - 0x98) << 15) + 0x20_00;
+            }
+        }
+        return text;
     }
-    $.$mol_charset_decode = $mol_charset_decode;
+    $.$mol_charset_ucf_decode = $mol_charset_ucf_decode;
 })($ || ($ = {}));
 
 ;
@@ -2465,7 +2490,7 @@ var $;
                 const len_max = val.length * 3;
                 const len_size = calc_size(len_max);
                 acquire(len_max);
-                const len = $mol_charset_encode_to(val, this.array, pos + len_size);
+                const len = $mol_charset_ucf_encode_to(val, this.array, pos + len_size);
                 dump_unum($mol_vary_tip.text, len, len_max);
                 pos += len;
                 release(len_max - len);
@@ -2674,7 +2699,7 @@ var $;
             };
             const read_text = (kind) => {
                 const len = read_unum(kind);
-                const text = $mol_charset_decode(new Uint8Array(array.buffer, array.byteOffset + pos, len));
+                const text = $mol_charset_ucf_decode(new Uint8Array(array.buffer, array.byteOffset + pos, len));
                 pos += len;
                 stream.push(text);
                 return text;
@@ -4451,27 +4476,21 @@ var $;
 ;
 "use strict";
 var $;
-(function ($) {
-    $mol_test({
-        'encode empty'() {
-            $mol_assert_equal($mol_charset_encode(''), new Uint8Array([]));
-        },
-        'encode 1 octet'() {
-            $mol_assert_equal($mol_charset_encode('F'), new Uint8Array([0x46]));
-        },
-        'encode 2 octet'() {
-            $mol_assert_equal($mol_charset_encode('Б'), new Uint8Array([0xd0, 0x91]));
-        },
-        'encode 3 octet'() {
-            $mol_assert_equal($mol_charset_encode('ह'), new Uint8Array([0xe0, 0xa4, 0xb9]));
-        },
-        'encode 4 octet'() {
-            $mol_assert_equal($mol_charset_encode('𐍈'), new Uint8Array([0xf0, 0x90, 0x8d, 0x88]));
-        },
-        'encode surrogate pair'() {
-            $mol_assert_equal($mol_charset_encode('😀'), new Uint8Array([0xf0, 0x9f, 0x98, 0x80]));
-        },
-    });
+(function ($_1) {
+    var $$;
+    (function ($$) {
+        $mol_test({
+            "Complex UCF encoding"($) {
+                $mol_assert_equal($mol_charset_ucf_encode('hi мир, 美しい 世界 🏴‍☠\t\n'), new Uint8Array([
+                    0x68, 0x69, 0x20,
+                    0xA4, 0x3C, 0x38, 0x40, 0x8B, 0x95,
+                    0x98, 0x0E, 0xBF, 0xFC, 0x57, 0x44, 0x95,
+                    0x98, 0x16, 0x5C, 0x4C, 0xAA, 0x95,
+                    0x9B, 0x74, 0xA7, 0xDC, 0x0D, 0xE8, 0x20, 0x9C, 0x09, 0x0A,
+                ]));
+            },
+        });
+    })($$ = $_1.$$ || ($_1.$$ = {}));
 })($ || ($ = {}));
 
 ;
@@ -4517,19 +4536,21 @@ var $;
 ;
 "use strict";
 var $;
-(function ($) {
-    $mol_test({
-        'decode utf8 string'() {
-            const str = 'Hello, ΧΨΩЫ';
-            const encoded = new Uint8Array([72, 101, 108, 108, 111, 44, 32, 206, 167, 206, 168, 206, 169, 208, 171]);
-            $mol_assert_equal($mol_charset_decode(encoded), str);
-            $mol_assert_equal($mol_charset_decode(encoded, 'utf8'), str);
-        },
-        'decode empty string'() {
-            const encoded = new Uint8Array([]);
-            $mol_assert_equal($mol_charset_decode(encoded), '');
-        },
-    });
+(function ($_1) {
+    var $$;
+    (function ($$) {
+        $mol_test({
+            "Complex UCF eecoding"($) {
+                $mol_assert_equal('hi мир, 美しい 世界 🏴‍☠\t\n', $mol_charset_ucf_decode(new Uint8Array([
+                    0x68, 0x69, 0x20,
+                    0xA4, 0x3C, 0x38, 0x40, 0x8B, 0x95,
+                    0x98, 0x0E, 0xBF, 0xFC, 0x57, 0x44, 0x95,
+                    0x98, 0x16, 0x5C, 0x4C, 0xAA, 0x95,
+                    0x9B, 0x74, 0xA7, 0xDC, 0x0D, 0xE8, 0x20, 0x9C, 0x09, 0x0A,
+                ])));
+            },
+        });
+    })($$ = $_1.$$ || ($_1.$$ = {}));
 })($ || ($ = {}));
 
 ;
@@ -4887,7 +4908,7 @@ var $;
         const { uint, link, spec, blob, text, list, tupl, sint } = $mol_vary_tip;
         const { none, both, fp16, fp32, fp64 } = $mol_vary_spec;
         const { L1, L2, L4, L8, LA } = $mol_vary_len;
-        const str = $mol_charset_encode;
+        const str = $mol_charset_ucf_encode;
         function check(vary, ideal, Vary = $mol_vary) {
             const pack = Vary.pack(vary);
             $mol_assert_equal(Vary.take(pack), vary);
@@ -4976,11 +4997,11 @@ var $;
             },
             "vary pack text"($) {
                 check(['foo'], [text | 3, ...str('foo')]);
-                check(['абв'], [text | 6, ...str('абв')]);
+                check(['абв'], [text | 4, ...str('абв')]);
                 const long_lat = 'abcdefghijklmnopqrst';
                 check([long_lat], [text | L1, 20, ...str(long_lat)]);
                 const long_cyr = 'абвгдеёжзийклмнопрст';
-                check([long_cyr], [text | L1, 40, ...str(long_cyr)]);
+                check([long_cyr], [text | L1, 21, ...str(long_cyr)]);
             },
             "vary pack dedup text"($) {
                 check([["f", "f"]], [list | 2, text | 1, ...str('f'), link | 0]);