json-as 1.1.26 → 1.2.0

package/assembly/deserialize/swar/string.ts

@@ -1,3 +1,4 @@
+import { bs } from "../../../lib/as-bs";
 import { BACK_SLASH } from "../../custom/chars";
 import { DESERIALIZE_ESCAPE_TABLE, ESCAPE_HEX_TABLE } from "../../globals/tables";
 
@@ -23,68 +24,69 @@ import { DESERIALIZE_ESCAPE_TABLE, ESCAPE_HEX_TABLE } from "../../globals/tables
  * @returns number of bytes written
  */
 // todo: optimize and stuff. it works, its not pretty. ideally, i'd like this to be (nearly) branchless
-export function deserializeString_SWAR(srcStart: usize, srcEnd: usize, dst: usize): usize {
+export function deserializeString_SWAR(srcStart: usize, srcEnd: usize): void {
   srcStart += 2;
   srcEnd -= 2;
-  let dst_ptr = changetype<usize>(dst);
-  const src_end_15 = srcEnd - 15;
+  bs.proposeSize(u32(srcEnd - srcStart));
 
-  while (srcStart < src_end_15) {
-    const block = v128.load(srcStart);
-    v128.store(dst_ptr, block);
+//   const src_end_15 = srcEnd - 15;
+//   while (srcStart < src_end_15) {
+//     const block = load<u64>(srcStart);
+//     store<u64>(dst_ptr, block);
 
-    const backslash_indices = i16x8.eq(block, SPLAT_92);
-    let mask = i16x8.bitmask(backslash_indices);
+//     let mask = v64x4_eq(block, BACKSLASH_MASK);
 
-    while (mask != 0) {
-      const lane_index = ctz(mask) << 1;
-      const dst_offset = dst_ptr + lane_index;
-      const src_offset = srcStart + lane_index;
-      const code = load<u16>(src_offset, 2);
+//     while (mask != 0) {
+//       const lane_index = usize(ctz(mask) >> 3);
+//       const dst_offset = dst_ptr + lane_index;
+//       const src_offset = srcStart + lane_index;
+//       const code = load<u16>(src_offset, 2);
 
-      mask &= mask - 1;
-      if (code == 117 && load<u32>(src_offset, 4) == 3145776) {
-        const block = load<u32>(src_offset, 8);
-        const codeA = block & 0xffff;
-        const codeB = (block >> 16) & 0xffff;
-        const escapedA = load<u8>(ESCAPE_HEX_TABLE + codeA);
-        const escapedB = load<u8>(ESCAPE_HEX_TABLE + codeB);
-        const escaped = (escapedA << 4) + escapedB;
-        // console.log("Escaped:");
-        // console.log("  a: " + escapedA.toString())
-        // console.log("  b: " + escapedB.toString());
-        // console.log("  c: " + escaped.toString());
-        // console.log("  o: " + (dst_ptr - dst).toString());
-        // console.log("  d: " + (dst_offset - dst).toString())
-        // console.log("  l: " + (lane_index).toString())
-        store<u16>(dst_offset, escaped);
-        v128.store(dst_offset, v128.load(src_offset, 4), 2);
-        if (lane_index >= 6) {
-          const bytes_left = lane_index - 4;
-          srcStart += bytes_left;
-          dst_ptr += bytes_left;
-          // console.log("  e: " + (bytes_left).toString())
-        }
-        dst_ptr -= 10;
-      } else {
-        const escaped = load<u8>(DESERIALIZE_ESCAPE_TABLE + code);
-        store<u16>(dst_offset, escaped);
-        v128.store(dst_offset, v128.load(src_offset, 4), 2);
-        // console.log("Escaped:");
-        if (lane_index == 14) {
-          srcStart += 2;
-        } else {
-          dst_ptr -= 2;
-        }
-      }
-    }
+//       mask &= mask - 1;
+//       if (code == 117 && load<u32>(src_offset, 4) == 3145776) {
+//         const block = load<u32>(src_offset, 8);
+//         const codeA = block & 0xffff;
+//         const codeB = (block >> 16) & 0xffff;
+//         const escapedA = load<u8>(ESCAPE_HEX_TABLE + codeA);
+//         const escapedB = load<u8>(ESCAPE_HEX_TABLE + codeB);
+//         const escaped = (escapedA << 4) + escapedB;
+//         // console.log("Escaped:");
+//         // console.log("  a: " + escapedA.toString())
+//         // console.log("  b: " + escapedB.toString());
+//         // console.log("  c: " + escaped.toString());
+//         // console.log("  o: " + (dst_ptr - dst).toString());
+//         // console.log("  d: " + (dst_offset - dst).toString())
+//         // console.log("  l: " + (lane_index).toString())
+//         store<u16>(dst_offset, escaped);
+//         memory.copy(dst_offset + 2, src_offset + 4, (4 - lane_index) << 1);
+//         // v128.store(dst_offset, v128.load(src_offset, 4), 2);
+//         if (lane_index >= 6) {
+//           const bytes_left = lane_index - 4;
+//           srcStart += bytes_left;
+//           dst_ptr += bytes_left;
+//           // console.log("  e: " + (bytes_left).toString())
+//         }
+//         dst_ptr -= 10;
+//       } else {
+//         const escaped = load<u8>(DESERIALIZE_ESCAPE_TABLE + code);
+//         store<u16>(dst_offset, escaped);
+//         memory.copy(dst_offset + 2, src_offset + 4, (4 - lane_index) << 1);
+//         // v128.store(dst_offset, v128.load(src_offset, 4), 2);
+//         // console.log("Escaped:");
+//         if (lane_index == 14) {
+//           srcStart += 2;
+//         } else {
+//           dst_ptr -= 2;
+//         }
+//       }
+//     }
 
-    srcStart += 16;
-    dst_ptr += 16;
+//     srcStart += 16;
+//     dst_ptr += 16;
 
-    // console.log("src: " + (srcStart - changetype<usize>(src)).toString());
-    // console.log("dst: " + (dst_ptr - dst).toString());
-  }
+//     // console.log("src: " + (srcStart - changetype<usize>(src)).toString());
+//     // console.log("dst: " + (dst_ptr - dst).toString());
+//   }
   while (srcStart < srcEnd) {
     let code = load<u16>(srcStart);
     if (code == BACK_SLASH) {
@@ -96,22 +98,20 @@ export function deserializeString_SWAR(srcStart: usize, srcEnd: usize, dst: usiz
         const escapedA = load<u8>(ESCAPE_HEX_TABLE + codeA);
         const escapedB = load<u8>(ESCAPE_HEX_TABLE + codeB);
         const escaped = (escapedA << 4) + escapedB;
-        store<u16>(dst_ptr, escaped);
-        dst_ptr += 2;
+        store<u16>(bs.offset, escaped);
+        bs.offset += 2;
         srcStart += 12;
       } else {
-        store<u16>(dst_ptr, code);
-        dst_ptr += 2;
+        store<u16>(bs.offset, code);
+        bs.offset += 2;
         srcStart += 4;
       }
     } else {
-      store<u16>(dst_ptr, code);
-      dst_ptr += 2;
+      store<u16>(bs.offset, code);
+      bs.offset += 2;
       srcStart += 2;
     }
   }
-
-  return dst_ptr - dst;
 }
 
 // @ts-ignore: decorators allowed

package/assembly/index.ts

@@ -11,7 +11,7 @@ import { deserializeFloat } from "./deserialize/simple/float";
 import { deserializeMap } from "./deserialize/simple/map";
 import { deserializeDate } from "./deserialize/simple/date";
 import { deserializeInteger } from "./deserialize/simple/integer";
-import { deserializeString } from "./deserialize/simple/string";
+// import { deserializeString } from "./deserialize/simple/string";
 import { serializeArbitrary } from "./serialize/simple/arbitrary";
 
 import { NULL_WORD, QUOTE } from "./custom/chars";
@@ -29,6 +29,8 @@ import { serializeRaw } from "./serialize/simple/raw";
 import { deserializeRaw } from "./deserialize/simple/raw";
 import { serializeString_SIMD } from "./serialize/simd/string";
 import { serializeString_SWAR } from "./serialize/swar/string";
+import { deserializeString_SWAR } from "./deserialize/swar/string";
+import { deserializeString } from "./deserialize/simple/string";
 // import { deserializeString_SIMD } from "./deserialize/simd/string";
 
 /**
@@ -185,8 +187,9 @@ export namespace JSON {
       //   // @ts-ignore
       //   return changetype<string>(deserializeString_SIMD(dataPtr, dataPtr + dataSize, __new(dataSize - 4, idof<string>())));
       // } else {
+      
       // @ts-ignore
-      return deserializeString(dataPtr, dataPtr + dataSize, __new(dataSize - 4, idof<string>()));
+      return deserializeString(dataPtr, dataPtr + dataSize, 0);
       // }
     } else if (isArray<T>()) {
       // @ts-ignore
@@ -605,6 +608,7 @@ export namespace JSON {
       return deserializeFloat<T>(srcStart, srcEnd);
     } else if (isString<T>()) {
       if (srcEnd - srcStart < 4) throw new Error("Cannot parse data as string because it was formatted incorrectly!");
+      
       // @ts-ignore: type
       return deserializeString(srcStart, srcEnd, dst);
     } else if (isNullable<T>() && srcEnd - srcStart == 8 && load<u64>(srcStart) == 30399761348886638) {

package/assembly/serialize/simd/string.ts

@@ -37,7 +37,7 @@ export function serializeString_SIMD(src: string): void {
 
     while (mask != 0) {
       const lane_index = ctz(mask) << 1;
-      // console.log("lane index "  + lane_index.toString());
+      // console.log("lane: "  + (lane_index >= 8 ? (lane_index - 8).toString():lane_index.toString()));
       const src_offset = srcStart + lane_index;
       const code = load<u16>(src_offset) << 2;
       const escaped = load<u32>(SERIALIZE_ESCAPE_TABLE + code);

package/assembly/serialize/swar/string.ts

@@ -7,9 +7,11 @@ import { bytes } from "../../util/bytes";
 // @ts-ignore: decorator allowed
 @lazy const LANE_MASK_HIGH = 0xFF00_FF00_FF00_FF00;
 // @ts-ignore: decorator allowed
+@lazy const ONES: u64 = 0x0101010101010101;
+// @ts-ignore: decorator allowed
 @lazy const LANE_MASK_LOW = 0x00FF_00FF_00FF_00FF;
 // @ts-ignore: decorator allowed
-@lazy const UINT64_H8 = 0x8080808080808080;
+@lazy const HIGHS = 0x8080808080808080;
 // @ts-ignore: decorator allowed
 @lazy const QUOTE_MASK = 0x0022_0022_0022_0022;
 // @ts-ignore: decorator allowed
@@ -33,44 +35,40 @@ export function serializeString_SWAR(src: string): void {
     const block = load<u64>(srcStart);
     store<u64>(bs.offset, block);
 
-    let mask = (
-      v64x4_eq(block, QUOTE_MASK) |
-      v64x4_eq(block, BACKSLASH_MASK) |
-      v64x4_ltu(block & LANE_MASK_LOW, CONTROL_MASK)
-    ) & UINT64_H8
+    let mask = v64x4_should_escape(block);
 
     while (mask != 0) {
-      const lane_index = usize(ctz(mask) >> 3);
-      // console.log("lane index " + lane_index.toString());
+      const lane_index = usize(ctz(mask) >> 3); // 0 2 4 6
       const src_offset = srcStart + lane_index;
+      // const dst_offset = bs.offset + lane_index;
       const code = load<u16>(src_offset) << 2;
+      // console.log("lane: " + lane_index.toString())
       const escaped = load<u32>(SERIALIZE_ESCAPE_TABLE + code);
 
+      mask = mask & ~(0xFF << (lane_index << 3));
       if ((escaped & 0xffff) != BACK_SLASH) {
         bs.growSize(10);
         const dst_offset = bs.offset + lane_index;
         store<u64>(dst_offset, U00_MARKER);
         store<u32>(dst_offset, escaped, 8);
-        // store<u64>(dst_offset, load<u64>(src_offset, 2), 12); // unsafe. can overflow here
-        memory.copy(dst_offset + 12, src_offset + 2, (4 - lane_index) << 1);
+        store<u64>(dst_offset, load<u64>(src_offset, 2), 12); // unsafe. can overflow here
+        // memory.copy(dst_offset + 12, src_offset + 2, (4 - lane_index) << 1);
         bs.offset += 10;
       } else {
         bs.growSize(2);
         const dst_offset = bs.offset + lane_index;
         store<u32>(dst_offset, escaped);
-        // store<u64>(dst_offset, load<u64>(src_offset, 2), 4);
-        memory.copy(dst_offset + 4, src_offset + 2, (4 - lane_index) << 1);
+        store<u64>(dst_offset, load<u64>(src_offset, 2), 4);
+        // memory.copy(dst_offset + 4, src_offset + 2, (4 - lane_index) << 1);
         bs.offset += 2;
       }
-
-      mask &= mask - 1;
     }
 
     srcStart += 8;
     bs.offset += 8;
   }
 
-  while (srcStart <= srcEnd - 2) {
+while (srcStart <= srcEnd - 2) {
     const code = load<u16>(srcStart);
     if (code == 92 || code == 34 || code < 32) {
       const escaped = load<u32>(SERIALIZE_ESCAPE_TABLE + (code << 2));
@@ -96,14 +94,16 @@ export function serializeString_SWAR(src: string): void {
 }
 
 // @ts-ignore: decorators allowed
-@inline function v64x4_eq(x: u64, y: u64): u64 {
-  const xored = (x ^ LANE_MASK_HIGH) ^ y;
-  const mask = (((xored >> 1) | UINT64_H8) - xored) & UINT64_H8;
-  return (mask << 1) - (mask >> 7);
-}
-
-// @ts-ignore: decorators allowed
-@inline function v64x4_ltu(a: u64, b: u64): u64 {
-  // Vigna's algorithm - fastest SWAR unsigned less-than
-  return (((a | UINT64_H8) - (b & ~UINT64_H8)) | (a ^ b)) ^ (a | ~b);
+@inline function v64x4_should_escape(x: u64): u64 {
+  x = x ^ 0xFF00FF00FF00FF00;
+  
+  const is_ascii: u64 = HIGHS & ~x;
+  const xor2: u64 = x ^ 0x0202020202020202;
+  const lt32_or_eq34: u64 = xor2 - 0x2121212121212121;
+  const sub92: u64 = x ^ 0x5C5C5C5C5C5C5C5C;
+  const eq92: u64 = sub92 - ONES;
+  
+  const high_bits: u64 = (lt32_or_eq34 | eq92) & is_ascii;
+  
+  return high_bits
 }

package/assembly/test.tmp.ts

@@ -1,11 +1,133 @@
+import {
+  JSON
+} from ".";
 import {
   bs
 } from "../lib/as-bs";
+import {
+  expect
+} from "./__tests__/lib";
 import {
   serializeString_SIMD
 } from "./serialize/simd/string";
 import {
   serializeString_SWAR
 } from "./serialize/swar/string";
-serializeString_SWAR("a\"cd");
-console.log(bs.out<string>());
+@lazy
+const ONES: u64 = 72340172838076673;
+@lazy
+const HIGHS: u64 = -9187201950435737472;
+@lazy
+const LANE_MASK_LOW: u64 = 71777214294589695;
+@inline
+function json_escapable_byte_mask(x: u64): u64 {
+  x = x ^ -71777214294589696;
+  const is_ascii: u64 = HIGHS & ~x;
+  const xor2: u64 = x ^ 144680345676153346;
+  const lt32_or_eq34: u64 = xor2 - 2387225703656530209;
+  const sub92: u64 = x ^ 6655295901103053916;
+  const eq92: u64 = sub92 - ONES;
+  const high_bits: u64 = (lt32_or_eq34 | eq92) & is_ascii;
+  return high_bits;
+}
+function to_u64(s: string): u64 {
+  return load<u64>(changetype<usize>(s));
+}
+export function mask_to_string(mask: u64): string {
+  let result = "0x";
+  for (let i = 7; i >= 0; i--) {
+    const byte = u8((mask >> (i * 8)) & 255);
+    const hi = (byte >> 4) & 15;
+    const lo = byte & 15;
+    result += hi < 10 ? String.fromCharCode(48 + hi) : String.fromCharCode(55 + hi);
+    result += lo < 10 ? String.fromCharCode(48 + lo) : String.fromCharCode(55 + lo);
+    result += " ";
+  }
+  return result;
+}
+function test_mask(input: string, expected_mask: u64, description: string): void {
+  const mask = json_escapable_byte_mask(to_u64(input));
+  const pass = mask == expected_mask;
+  console.log((pass ? "✓ " : "✗ ") + description);
+  if (!pass) {
+    console.log("  Input: \"" + input + "\"");
+    console.log("  Expected: " + mask_to_string(expected_mask));
+    console.log("  Got:      " + mask_to_string(mask));
+    process.exit(1);
+  }
+}
+console.log("=== No Escapes Needed ===");
+test_mask("abcd", 0, "Regular ASCII letters");
+test_mask("ABCD", 0, "Uppercase letters");
+test_mask("1234", 0, "Numbers");
+test_mask("test", 0, "Common word");
+test_mask("____", 0, "Underscores");
+test_mask("....", 0, "Periods");
+test_mask("!#$%", 0, "Safe symbols");
+test_mask("&*()", 0, "More symbols");
+test_mask("+,-.", 0, "Math symbols");
+console.log();
+console.log("=== Quote Character (34 / 0x22) ===");
+test_mask("\"abc", 128, "Quote at position 0");
+test_mask("a\"bc", 8388608, "Quote at position 1");
+test_mask("ab\"c", 549755813888, "Quote at position 2");
+test_mask("abc\"", 36028797018963968, "Quote at position 3");
+test_mask("\"\"\"\"", 36029346783166592, "All quotes");
+console.log();
+console.log("=== Backslash Character (92 / 0x5C) ===");
+test_mask("\babc", 128, "Backslash at position 0");
+test_mask("a\bbc", 8388608, "Backslash at position 1");
+test_mask("ab\bc", 549755813888, "Backslash at position 2");
+test_mask("abc\b", 36028797018963968, "Backslash at position 3");
+test_mask("\b\b\b\b", 36029346783166592, "All backslashes");
+console.log();
+console.log("=== Control Characters (< 32) ===");
+test_mask("\0abc", 128, "Null character at position 0");
+test_mask("\nabc", 128, "Newline at position 0");
+test_mask("\rabc", 128, "Carriage return at position 0");
+test_mask("\tabc", 128, "Tab at position 0");
+test_mask("a\nbc", 8388608, "Newline at position 1");
+test_mask("ab\nc", 549755813888, "Newline at position 2");
+test_mask("abc\n", 36028797018963968, "Newline at position 3");
+console.log();
+console.log("=== Boundary Cases ===");
+test_mask("abc", 128, "Character 31 (should escape)");
+test_mask(" abc", 0, "Space (32, should NOT escape)");
+test_mask("!abc", 0, "Character 33 (should NOT escape)");
+console.log();
+console.log("=== Mixed Scenarios ===");
+test_mask("\"\nabc", 128 | 8388608, "Quote + newline");
+test_mask("\"\babc", 128 | 8388608, "Quote + backslash");
+test_mask("\"\b\nabc", 128 | 8388608 | 549755813888, "Quote + backslash + newline");
+test_mask("a\"\tb", 8388608 | 549755813888, "Quote + tab at pos 1-2");
+console.log();
+console.log("=== Multiple Escapes ===");
+test_mask("\"\"\b\b", 8388736 | 36029346774777856, "All 4 positions need escape");
+test_mask("a\"b\b", 8388608 | 36028797018963968, "Quote at pos 1, backslash at pos 3");
+test_mask("\"a\bb", 128 | 549755813888, "Quote at pos 0, backslash at pos 2");
+console.log();
+console.log("=== Edge Cases ===");
+test_mask("
", 36029346783166592, "All control characters (1-4)");
+test_mask("  ", 128 | 8388608, "Characters 30-31, then two spaces");
+test_mask("]}~", 0, "High safe ASCII (125-127)");
+console.log();
+console.log("=== Specific JSON Escape Sequences ===");
+test_mask("\f\n\r", 36029346783166592, "Backspace, form feed, newline, carriage return");
+test_mask("\"\b\f", 36029346783166592, "Quote, backslash, backspace, form feed");
+console.log();
+function get_lane(mask: u64): u64 {
+  return ctz(mask) >> 3;
+}
+function clearLane(mask: u64, lane_index: u32): u64 {
+  return mask & ~(255 << (lane_index << 3));
+}
+expect(get_lane(128)).toBe(0);
+expect(get_lane(8388608)).toBe(2);
+expect(get_lane(549755813888)).toBe(4);
+expect(get_lane(36028797018963968)).toBe(6);
+let x: u64 = 36029346783166592;
+expect(get_lane(x = clearLane(x, 0))).toBe(2);
+expect(get_lane(x = clearLane(x, 2))).toBe(4);
+expect(get_lane(x = clearLane(x, 4))).toBe(6);
+serializeString_SWAR("\0
\t\n\v\f\rabcdefg");
+expect(bs.out<string>()).toBe("\"\bu0000\bu0001\bu0002\bu0003\bu0004\bu0005\bu0006\bu0007\bb\bt\bn\bu000b\bf\br\bu000eabcdefg\"");

package/assembly/test.ts

@@ -1,7 +0,0 @@
-import { bs } from "../lib/as-bs";
-import { serializeString_SIMD } from "./serialize/simd/string";
-import { serializeString_SWAR } from "./serialize/swar/string";
-
-serializeString_SWAR("a\"cd");
-// serializeString_SIMD("a\0b\"\nd");
-console.log(bs.out<string>());

package/bench/abc.bench.ts

@@ -1,4 +1,5 @@
-import { bench } from "./lib/bench.js";
+import { dumpToFile } from "./lib/bench.js";
+import { bench, blackbox } from "./lib/bench.js";
 
 const v1 = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
 const v2 = '"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"';
@@ -6,22 +7,19 @@ const v2 = '"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"';
 bench(
   "Serialize Alphabet",
   () => {
-    blackbox(JSON.stringify(blackbox(v1)));
+    blackbox(JSON.stringify(v1));
   },
   64_000_00,
   v1.length << 1,
 );
+dumpToFile("abc", "serialize")
 
 bench(
   "Deserialize Alphabet",
   () => {
-    blackbox(JSON.parse(blackbox(v2)));
+    blackbox(JSON.parse(v2));
   },
   64_000_00,
   v2.length << 1,
 );
-
-function blackbox<T>(value: T): T {
-  (globalThis as any).__blackhole = value;
-  return globalThis.__blackhole;
-}
+dumpToFile("abc", "deserialize")