@atproto/lex-json 0.0.15 → 0.0.16

package/src/lex-json.bench.ts

@@ -0,0 +1,125 @@
+import { bench, describe } from 'vitest'
+import { JsonValue } from './json.js'
+import {
+  LexParseOptions,
+  jsonToLex,
+  lexParse,
+  parseSpecialJsonObject,
+} from './lex-json.js'
+
+// This benchmark compares the performance of two implementations of
+// lexParse:
+// - One that uses a reviver function with JSON.parse to directly parse special
+//   objects and handle numbers (lexParse with reviver)
+// - One that first parses JSON to a plain JS object and then converts it to
+//   LexValue using jsonToLex (lexParse with jsonToLex)
+
+describe('small object', () => {
+  benchData({
+    $type: 'app.bsky.feed.post',
+    text: 'Hello world! 👋',
+    createdAt: '2024-01-01T00:00:00Z',
+  })
+})
+
+describe('simple mixed structure', () => {
+  benchData({
+    cid: {
+      $link: 'bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a',
+    },
+    bytes: {
+      $bytes: 'nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0',
+    },
+    blob: {
+      $type: 'blob',
+      ref: {
+        $link: 'bafkreig77vqcdozl2wyk6z3cscaj5q5fggi53aoh64fewkdiri3cdauyn4',
+      },
+      mimeType: 'image/jpeg',
+      size: 10000,
+    },
+    nested: {
+      array: [
+        {
+          number: 42,
+          string: 'hello world',
+          bool: true,
+          null: null,
+        },
+      ],
+      string: 'Hello 世界! 🌍🌎🌏 Ñoño',
+      createdAt: '2024-01-01T00:00:00Z',
+    },
+  })
+})
+
+describe('large payload', () => {
+  benchData({
+    items: Array.from({ length: 25 }, (_, i) => ({
+      id: i,
+      name: `Item ${i}`,
+      longUnicode:
+        'Lorem ipsum dolor sit amet, consectetur adipiscing elit 🤩.\n'.repeat(
+          2,
+        ),
+      tags: ['tag1', 'tag2', 'tag3'],
+      bytes: {
+        $bytes: Buffer.from(`This is some byte data for item ${i}`).toString(
+          'base64',
+        ),
+      },
+      cid: {
+        $link: 'bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a',
+      },
+      metadata: {
+        created: '2024-01-01T00:00:00Z',
+        count: i * 10,
+        nested: {
+          flag: i % 2 === 0,
+          values: [i, i * 2, i * 3],
+        },
+        items: Array.from({ length: 5 }, (_, j) => ({
+          id: `${i}-${j}`,
+          value: `Value ${i}-${j}`,
+        })),
+      },
+    })),
+  })
+})
+
+function benchData(data: unknown, options?: LexParseOptions) {
+  const jsonString = JSON.stringify(data)
+
+  const withReviver: typeof lexParse = (input, options = { strict: true }) => {
+    return JSON.parse(input, (key: string, value: JsonValue) => {
+      switch (typeof value) {
+        case 'object':
+          if (value === null) return null
+          if (Array.isArray(value)) return value
+          return parseSpecialJsonObject(value, options) ?? value
+        case 'number':
+          if (Number.isSafeInteger(value)) return value
+          if (options && options.strict === false) return value
+          throw new TypeError(`Invalid non-integer number: ${value}`)
+        default:
+          return value
+      }
+    })
+  }
+
+  const naiveParse: typeof lexParse = (input, options) => {
+    return jsonToLex(JSON.parse(input), options) as any
+  }
+
+  bench('current', () => {
+    lexParse(jsonString, options)
+  })
+
+  bench('with-reviver', () => {
+    withReviver(jsonString, options)
+  })
+
+  bench('naive', () => {
+    naiveParse(jsonString, options)
+  })
+}

package/src/lex-json.test.ts

@@ -336,6 +336,24 @@ export const acceptableVectors: Array<{
       $link: 'bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity',
     },
   },
+  {
+    note: 'blob with CBOR CID ref',
+    json: {
+      $type: 'blob',
+      ref: {
+        $link: 'bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a',
+      },
+      mimeType: 'image/png',
+      size: 1,
+    },
+  },
+  {
+    note: 'object with empty $type',
+    json: {
+      $type: '',
+      foo: 'bar',
+    },
+  },
 ]
 
 export const invalidVectors: Array<{
@@ -621,3 +639,353 @@ describe('lex > json > lex', () => {
     }
   })
 })
+
+describe('lexParseJsonBytes strict mode error parity with lexParse', () => {
+  describe('invalid JSON input throws SyntaxError containing "Unexpected token"', () => {
+    test('lexParse throws with Unexpected token', () => {
+      expect(() => lexParse('not valid json', { strict: true })).toThrow(
+        /Unexpected token/,
+      )
+    })
+
+    test('lexParseJsonBytes throws with Unexpected token', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from('not valid json'), { strict: true }),
+      ).toThrow(/Unexpected token/)
+    })
+
+    test('lexParseJsonBytes non-strict also throws with Unexpected token for invalid JSON', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from('not valid json'), { strict: false }),
+      ).toThrow(/Unexpected token/)
+    })
+  })
+
+  describe('float numbers: strict throws TypeError, non-strict accepts', () => {
+    const jsonStr = '{"value":1.5}'
+
+    test('lexParse strict throws TypeError with value in message', () => {
+      expect(() => lexParse(jsonStr, { strict: true })).toThrow(TypeError)
+      expect(() => lexParse(jsonStr, { strict: true })).toThrow(
+        'Invalid non-integer number: 1.5',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(jsonStr)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid non-integer number: 1.5',
+      )
+    })
+
+    test('lexParse non-strict accepts float', () => {
+      expect(() => lexParse(jsonStr, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict accepts float', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(jsonStr), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('exponent notation: safe integers accepted, unsafe integers rejected', () => {
+    test('lexParse strict accepts 1e10 (safe integer)', () => {
+      expect(lexParse('1e10', { strict: true })).toBe(1e10)
+    })
+
+    test('lexParseJsonBytes strict accepts 1e10 (safe integer)', () => {
+      expect(lexParseJsonBytes(Buffer.from('1e10'), { strict: true })).toBe(
+        1e10,
+      )
+    })
+
+    test('lexParse strict rejects 1e20 (unsafe integer)', () => {
+      expect(() => lexParse('1e20', { strict: true })).toThrow(TypeError)
+    })
+
+    test('lexParseJsonBytes strict rejects 1e20 (unsafe integer)', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from('1e20'), { strict: true }),
+      ).toThrow(TypeError)
+    })
+  })
+
+  describe('invalid blob: strict throws TypeError, non-strict returns plain object', () => {
+    const invalidBlobJson = '{"$type":"blob"}'
+
+    test('lexParse strict throws TypeError with "Invalid blob object"', () => {
+      expect(() => lexParse(invalidBlobJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(invalidBlobJson, { strict: true })).toThrow(
+        'Invalid blob object',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(invalidBlobJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid blob object',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() => lexParse(invalidBlobJson, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(invalidBlobJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('blob with CBOR CID: strict throws TypeError, non-strict returns plain object', () => {
+    const blobWithCborCidJson = JSON.stringify({
+      $type: 'blob',
+      ref: {
+        $link: 'bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a',
+      },
+      mimeType: 'image/png',
+      size: 1,
+    })
+
+    test('lexParse strict throws TypeError with "Invalid blob object"', () => {
+      expect(() => lexParse(blobWithCborCidJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(blobWithCborCidJson, { strict: true })).toThrow(
+        'Invalid blob object',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(blobWithCborCidJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid blob object',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() =>
+        lexParse(blobWithCborCidJson, { strict: false }),
+      ).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(blobWithCborCidJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('invalid $link: strict throws TypeError, non-strict returns plain object', () => {
+    const invalidLinkJson = '{"$link":"."}'
+
+    test('lexParse strict throws TypeError with "Invalid $link object"', () => {
+      expect(() => lexParse(invalidLinkJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(invalidLinkJson, { strict: true })).toThrow(
+        'Invalid $link object',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(invalidLinkJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid $link object',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() => lexParse(invalidLinkJson, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(invalidLinkJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('$link with extra fields: strict throws TypeError, non-strict returns plain object', () => {
+    const linkWithExtraJson =
+      '{"$link":"bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity","extra":"field"}'
+
+    test('lexParse strict throws TypeError with "Invalid $link object"', () => {
+      expect(() => lexParse(linkWithExtraJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(linkWithExtraJson, { strict: true })).toThrow(
+        'Invalid $link object',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(linkWithExtraJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid $link object',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() => lexParse(linkWithExtraJson, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(linkWithExtraJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('invalid $bytes: strict throws TypeError, non-strict returns plain object', () => {
+    const invalidBytesJson = '{"$bytes":"🐻"}'
+
+    test('lexParse strict throws TypeError with "Invalid $bytes object"', () => {
+      expect(() => lexParse(invalidBytesJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(invalidBytesJson, { strict: true })).toThrow(
+        'Invalid $bytes object',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(invalidBytesJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid $bytes object',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() => lexParse(invalidBytesJson, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(invalidBytesJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('$bytes with extra fields: strict throws TypeError, non-strict returns plain object', () => {
+    const bytesWithExtraJson =
+      '{"$bytes":"nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0","extra":"field"}'
+
+    test('lexParse strict throws TypeError with "Invalid $bytes object"', () => {
+      expect(() => lexParse(bytesWithExtraJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(bytesWithExtraJson, { strict: true })).toThrow(
+        'Invalid $bytes object',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(bytesWithExtraJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid $bytes object',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() =>
+        lexParse(bytesWithExtraJson, { strict: false }),
+      ).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(bytesWithExtraJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('empty $type: strict throws TypeError, non-strict returns plain object', () => {
+    const emptyTypeJson = '{"$type":"","foo":"bar"}'
+
+    test('lexParse strict throws TypeError with "Empty $type property"', () => {
+      expect(() => lexParse(emptyTypeJson, { strict: true })).toThrow(TypeError)
+      expect(() => lexParse(emptyTypeJson, { strict: true })).toThrow(
+        'Empty $type property',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(emptyTypeJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Empty $type property',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() => lexParse(emptyTypeJson, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(emptyTypeJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+
+  describe('non-string $type: strict throws TypeError, non-strict returns plain object', () => {
+    const nonStringTypeJson = '{"$type":123,"foo":"bar"}'
+
+    test('lexParse strict throws TypeError with type name in message', () => {
+      expect(() => lexParse(nonStringTypeJson, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParse(nonStringTypeJson, { strict: true })).toThrow(
+        'Invalid $type property (number)',
+      )
+    })
+
+    test('lexParseJsonBytes strict throws same TypeError', () => {
+      const bytes = Buffer.from(nonStringTypeJson)
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        TypeError,
+      )
+      expect(() => lexParseJsonBytes(bytes, { strict: true })).toThrow(
+        'Invalid $type property (number)',
+      )
+    })
+
+    test('lexParse non-strict returns plain object', () => {
+      expect(() => lexParse(nonStringTypeJson, { strict: false })).not.toThrow()
+    })
+
+    test('lexParseJsonBytes non-strict returns plain object', () => {
+      expect(() =>
+        lexParseJsonBytes(Buffer.from(nonStringTypeJson), { strict: false }),
+      ).not.toThrow()
+    })
+  })
+})

package/src/lex-json.ts

@@ -100,41 +100,26 @@ export function lexParse<T extends LexValue = LexValue>(
   input: string,
   options: LexParseOptions = { strict: false },
 ): T {
-  return JSON.parse(input, function (key: string, value: JsonValue): LexValue {
-    switch (typeof value) {
-      case 'object':
-        if (value === null) return null
-        if (Array.isArray(value)) return value
-        return parseSpecialJsonObject(value, options) ?? value
-      case 'number':
-        if (Number.isSafeInteger(value)) return value
-        if (options.strict) {
-          throw new TypeError(`Invalid non-integer number: ${value}`)
-        }
-      // fallthrough
-      default:
-        return value
-    }
-  })
+  // @NOTE see ./lex-json.bench.ts for performance comparison of implementation
+  // that uses a reviver function in JSON.parse vs. the current implementation.
+  return jsonToLex(JSON.parse(input), options) as T
 }
 
 /**
- * Parses a `Uint8Array` containing JSON data into a Lex value.
+ * Parses a JSON string from a byte array into Lex values.
  */
 export function lexParseJsonBytes(
-  jsonBytes: Uint8Array,
+  bytes: Uint8Array,
   options?: LexParseOptions,
 ): LexValue {
-  // @TODO optimize this to avoid intermediate string. This requires a custom
-  // JSON parser that can operate on binary data, which is non-trivial, but
-  // could be a future improvement if performance is a concern. See the link
-  // below for an example of a JSON parser that operates on binary data in
-  // @ipld/dag-json
-
-  // https://github.com/ipld/js-dag-json/blob/57912da6e9d64a179f7d2384c3b6d7b07fbfb143/src/index.js#L161
-
-  const jsonString = utf8FromBytes(jsonBytes)
-  return lexParse(jsonString, options)
+  // @NOTE see ./json-bytes-decoder.bench.ts for performance comparison of
+  // implementation that uses a decoder class that operates directly on bytes
+  // vs. the current implementation that first decodes bytes to string and then
+  // parses JSON. For more common cases, it seems that the trivial
+  // implementation works better than the decoder based solution, while having a
+  // small overhead for slower cases (~2% difference). Because of this, we keep
+  // the trivial implementation:
+  return lexParse(utf8FromBytes(bytes), options)
 }
 
 /**
@@ -181,10 +166,8 @@ export function jsonToLex(
     }
     case 'number':
       if (Number.isSafeInteger(value)) return value
-      if (options.strict) {
-        throw new TypeError(`Invalid non-integer number: ${value}`)
-      }
-    // fallthrough
+      if (options.strict === false) return value
+      throw new TypeError(`Invalid non-integer number: ${value}`)
     case 'boolean':
     case 'string':
       return value
@@ -327,7 +310,10 @@ function encodeLexMap(input: LexMap): JsonObject {
   return copy ?? (input as JsonObject)
 }
 
-function parseSpecialJsonObject(
+/**
+ * @internal
+ */
+export function parseSpecialJsonObject(
   input: LexMap,
   options: LexParseOptions,
 ): Cid | Uint8Array | BlobRef | undefined {

package/tsconfig.build.json

@@ -1,7 +1,7 @@
 {
   "extends": ["../../../tsconfig/isomorphic.json"],
   "include": ["./src"],
-  "exclude": ["**/*.test.ts"],
+  "exclude": ["**/*.bench.ts", "**/*.test.ts"],
   "compilerOptions": {
     "noImplicitAny": true,
     "importHelpers": true,

package/tsconfig.tests.json

@@ -1,6 +1,6 @@
 {
   "extends": "../../../tsconfig/vitest.json",
-  "include": ["./tests", "./src/**/*.test.ts"],
+  "include": ["./tests", "./src/**/*.bench.ts", "./src/**/*.test.ts"],
   "compilerOptions": {
     "noImplicitAny": true,
     "rootDir": "./",