@atproto/lex-data 0.1.3 → 0.1.4

package/src/utf8-from-base64.test.ts

@@ -1,39 +0,0 @@
-import { assert, describe, expect, it } from 'vitest'
-import {
-  utf8FromBase64Node,
-  utf8FromBase64Ponyfill,
-} from './utf8-from-base64.js'
-
-const strings = [
-  'Hello, World!',
-  '¡Hola, Mundo!',
-  'こんにちは世界',
-  '😀👩‍💻🌍',
-  '',
-  '𓀀𓁐𓂀𓃰𓄿𓅱𓆑𓇋𓈖𓉔𓊃𓋴𓌳𓍿𓎛𓏏',
-]
-
-for (const utf8FromBase64 of [
-  utf8FromBase64Node,
-  utf8FromBase64Ponyfill,
-] as const) {
-  assert(utf8FromBase64, 'implementation should not be null')
-
-  describe(utf8FromBase64, () => {
-    it('decodes base64 to utf8 string', () => {
-      for (const text of strings) {
-        const b64 = Buffer.from(text, 'utf8').toString('base64')
-        const decoded = utf8FromBase64(b64, 'base64')
-        expect(decoded).toBe(text)
-      }
-    })
-
-    it('decodes base64url to utf8 string', () => {
-      for (const text of strings) {
-        const b64u = Buffer.from(text, 'utf8').toString('base64url')
-        const decoded = utf8FromBase64(b64u, 'base64url')
-        expect(decoded).toBe(text)
-      }
-    })
-  })
-}

package/src/utf8-from-base64.ts

@@ -1,23 +0,0 @@
-import { fromString } from 'uint8arrays/from-string'
-import { NodeJSBuffer } from './lib/nodejs-buffer.js'
-import { Base64Alphabet } from './uint8array-base64.js'
-
-const Buffer = NodeJSBuffer
-
-export const utf8FromBase64Node = Buffer
-  ? function utf8FromBase64Node(
-      b64: string,
-      alphabet: Base64Alphabet = 'base64',
-    ): string {
-      return Buffer.from(b64, alphabet).toString('utf8')
-    }
-  : /* v8 ignore next -- @preserve */ null
-
-const textDecoder = /*#__PURE__*/ new TextDecoder()
-export function utf8FromBase64Ponyfill(
-  b64: string,
-  alphabet?: Base64Alphabet,
-): string {
-  const bytes = fromString(b64, alphabet)
-  return textDecoder.decode(bytes)
-}

package/src/utf8-from-bytes.test.ts

@@ -1,43 +0,0 @@
-import { assert, describe, expect, it } from 'vitest'
-import { utf8FromBytesNative, utf8FromBytesNode } from './utf8-from-bytes.js'
-
-for (const utf8FromBytes of [utf8FromBytesNode, utf8FromBytesNative] as const) {
-  assert(utf8FromBytes, 'utf8FromBytes implementation should not be null')
-  describe(utf8FromBytes, () => {
-    it('decodes empty Uint8Array', () => {
-      const decoded = utf8FromBytes(new Uint8Array(0))
-      expect(typeof decoded).toBe('string')
-      expect(decoded).toBe('')
-    })
-
-    it('decodes 10MB', () => {
-      const bytes = Buffer.allocUnsafe(10_000_000).fill('🐩')
-      const decoded = utf8FromBytes(bytes)
-      expect(decoded).toBe('🐩'.repeat(10_000_000 / 4))
-    })
-
-    for (const string of [
-      '',
-      '\0\0',
-      '\0\0\0',
-      '\0\0\0\0',
-      '__',
-      'é',
-      'àç',
-      '\0éàç',
-      '```\x1b',
-      'aaa',
-      'Hello, World!',
-      '😀😃😄😁😆😅😂🤣😊😇',
-      '👩‍💻👨‍💻👩‍🔬👨‍🔬👩‍🚀👨‍🚀',
-      '🌍🌎🌏🌐🪐🌟✨⚡🔥💧',
-    ] as const) {
-      const buffer = Buffer.from(string, 'utf8')
-
-      it(`decodes ${JSON.stringify(string)}`, () => {
-        const decoded = utf8FromBytes(buffer)
-        expect(decoded).toBe(string)
-      })
-    }
-  })
-}

package/src/utf8-from-bytes.ts

@@ -1,21 +0,0 @@
-import { NodeJSBuffer } from './lib/nodejs-buffer.js'
-
-const Buffer = NodeJSBuffer
-
-export const utf8FromBytesNode = Buffer
-  ? function utf8FromBytesNode(bytes: Uint8Array): string {
-      // @NOTE Buffer.from(bytes) creates a copy of the ArrayBuffer. The following
-      // allows us to avoid the copy by creating a Buffer that shares the same
-      // memory as the input Uint8Array.
-      const buffer = Buffer.from(
-        bytes.buffer,
-        bytes.byteOffset,
-        bytes.byteLength,
-      )
-      return buffer.toString('utf8')
-    }
-  : /* v8 ignore next -- @preserve */ null
-
-export function utf8FromBytesNative(bytes: Uint8Array): string {
-  return new TextDecoder('utf-8').decode(bytes)
-}

package/src/utf8-grapheme-len.test.ts

@@ -1,38 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import { graphemeLenNative, graphemeLenPonyfill } from './utf8-grapheme-len.js'
-
-describe(graphemeLenNative!, () => {
-  it('computes grapheme length', () => {
-    expect(graphemeLenNative!('a')).toBe(1)
-    expect(graphemeLenNative!('~')).toBe(1)
-    expect(graphemeLenNative!('ö')).toBe(1)
-    expect(graphemeLenNative!('ñ')).toBe(1)
-    expect(graphemeLenNative!('©')).toBe(1)
-    expect(graphemeLenNative!('⽘')).toBe(1)
-    expect(graphemeLenNative!('☎')).toBe(1)
-    expect(graphemeLenNative!('𓋓')).toBe(1)
-    expect(graphemeLenNative!('😀')).toBe(1)
-    expect(graphemeLenNative!('👨‍👩‍👧‍👧')).toBe(1)
-    expect(graphemeLenNative!('a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧')).toBe(10)
-    // https://github.com/bluesky-social/atproto/issues/4321
-    expect(graphemeLenNative!('नमस्ते')).toBe(3)
-  })
-})
-
-describe(graphemeLenPonyfill, () => {
-  it('computes grapheme length', () => {
-    expect(graphemeLenPonyfill('a')).toBe(1)
-    expect(graphemeLenPonyfill('~')).toBe(1)
-    expect(graphemeLenPonyfill('ö')).toBe(1)
-    expect(graphemeLenPonyfill('ñ')).toBe(1)
-    expect(graphemeLenPonyfill('©')).toBe(1)
-    expect(graphemeLenPonyfill('⽘')).toBe(1)
-    expect(graphemeLenPonyfill('☎')).toBe(1)
-    expect(graphemeLenPonyfill('𓋓')).toBe(1)
-    expect(graphemeLenPonyfill('😀')).toBe(1)
-    expect(graphemeLenPonyfill('👨‍👩‍👧‍👧')).toBe(1)
-    expect(graphemeLenPonyfill('a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧')).toBe(10)
-    // https://github.com/bluesky-social/atproto/issues/4321
-    expect(graphemeLenPonyfill('नमस्ते')).toBe(3)
-  })
-})

package/src/utf8-grapheme-len.ts

@@ -1,21 +0,0 @@
-import { countGraphemes } from 'unicode-segmenter/grapheme'
-
-// @TODO: Drop usage of "unicode-segmenter" package when Intl.Segmenter is
-// widely supported.
-// https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Global_Objects/Intl/Segmenter
-const segmenter =
-  'Segmenter' in Intl && typeof Intl.Segmenter === 'function'
-    ? /*#__PURE__*/ new Intl.Segmenter()
-    : /* v8 ignore next -- @preserve */ null
-
-export const graphemeLenNative = segmenter
-  ? function graphemeLenNative(str: string): number {
-      let length = 0
-      for (const _ of segmenter.segment(str)) length++
-      return length
-    }
-  : /* v8 ignore next -- @preserve */ null
-
-export function graphemeLenPonyfill(str: string): number {
-  return countGraphemes(str)
-}

package/src/utf8-len.test.ts

@@ -1,21 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import { utf8LenCompute, utf8LenNode } from './utf8-len.js'
-
-describe.each([utf8LenNode!, utf8LenCompute!] as const)('%o', (utf8Len) => {
-  it('computes utf8 string length', () => {
-    expect(utf8Len('a')).toBe(1)
-    expect(utf8Len('~')).toBe(1)
-    expect(utf8Len('ö')).toBe(2)
-    expect(utf8Len('ñ')).toBe(2)
-    expect(utf8Len('©')).toBe(2)
-    expect(utf8Len('⽘')).toBe(3)
-    expect(utf8Len('☎')).toBe(3)
-    expect(utf8Len('𓋓')).toBe(4)
-    expect(utf8Len('😀')).toBe(4)
-    expect(utf8Len('👨‍👩‍👧‍👧')).toBe(25)
-    // high surrogate with no low surrogate
-    expect(utf8Len('\uD83D')).toBe(3)
-    // low surrogate with no high surrogate
-    expect(utf8Len('\uDC00')).toBe(3)
-  })
-})

package/src/utf8-len.ts

@@ -1,51 +0,0 @@
-import { NodeJSBuffer } from './lib/nodejs-buffer.js'
-
-// @NOTE This file is not meant to be exported directly. Instead, we re-export
-// public functions from ./utf8.ts. The reason for this separation is that this
-// file allows to test both the NodeJS-optimized and ponyfill implementations.
-
-export const utf8LenNode = NodeJSBuffer
-  ? function utf8LenNode(string: string): number {
-      return NodeJSBuffer!.byteLength(string, 'utf8')
-    }
-  : /* v8 ignore next -- @preserve */ null
-
-export function utf8LenCompute(string: string): number {
-  // The code below is similar to TextEncoder's implementation of UTF-8
-  // encoding. However, using TextEncoder to get the byte length is slower
-  // as it requires allocating a new Uint8Array and copying data:
-
-  // return new TextEncoder().encode(string).byteLength
-
-  // The base length is the string length (all ASCII)
-  let len = string.length
-  let code: number
-
-  // The loop calculates the number of additional bytes needed for
-  // non-ASCII characters
-  for (let i = 0; i < string.length; i += 1) {
-    code = string.charCodeAt(i)
-
-    if (code <= 0x7f) {
-      // ASCII, 1 byte
-    } else if (code <= 0x7ff) {
-      // 2 bytes char
-      len += 1
-    } else {
-      // 3 bytes char
-      len += 2
-      // If the current char is a high surrogate, and the next char is a low
-      // surrogate, skip the next char as the total is a 4 bytes char
-      // (represented as a surrogate pair in UTF-16) and was already accounted
-      // for.
-      if (code >= 0xd800 && code <= 0xdbff) {
-        code = string.charCodeAt(i + 1)
-        if (code >= 0xdc00 && code <= 0xdfff) {
-          i++
-        }
-      }
-    }
-  }
-
-  return len
-}

package/src/utf8-to-base64.test.ts

@@ -1,35 +0,0 @@
-import { assert, describe, expect, it } from 'vitest'
-import { utf8ToBase64Node, utf8ToBase64Ponyfill } from './utf8-to-base64.js'
-
-const strings = [
-  'Hello, World!',
-  '¡Hola, Mundo!',
-  'こんにちは世界',
-  '😀👩‍💻🌍',
-  '',
-  '𓀀𓁐𓂀𓃰𓄿𓅱𓆑𓇋𓈖𓉔𓊃𓋴𓌳𓍿𓎛𓏏',
-]
-
-for (const utf8ToBase64 of [utf8ToBase64Node, utf8ToBase64Ponyfill] as const) {
-  assert(utf8ToBase64, 'implementation should not be null')
-
-  describe(utf8ToBase64, () => {
-    it('encodes utf8 string to base64', () => {
-      for (const text of strings) {
-        const b64 = Buffer.from(text, 'utf8')
-          .toString('base64')
-          .replaceAll('=', '') // utf8ToBase64 omits padding
-        const encoded = utf8ToBase64(text, 'base64')
-        expect(encoded).toBe(b64)
-      }
-    })
-
-    it('encodes utf8 string to base64url', () => {
-      for (const text of strings) {
-        const b64u = Buffer.from(text, 'utf8').toString('base64url')
-        const encoded = utf8ToBase64(text, 'base64url')
-        expect(encoded).toBe(b64u)
-      }
-    })
-  })
-}

package/src/utf8-to-base64.ts

@@ -1,22 +0,0 @@
-import { toString } from 'uint8arrays/to-string'
-import { NodeJSBuffer } from './lib/nodejs-buffer.js'
-import { Base64Alphabet } from './uint8array-base64.js'
-import { toBase64Node } from './uint8array-to-base64.js'
-
-const Buffer = NodeJSBuffer
-
-export const utf8ToBase64Node = Buffer
-  ? function utf8ToBase64Node(text: string, alphabet?: Base64Alphabet): string {
-      const buffer = Buffer.from(text, 'utf8')
-      return toBase64Node!(buffer, alphabet)
-    }
-  : /* v8 ignore next -- @preserve */ null
-
-const textEncoder = /*#__PURE__*/ new TextEncoder()
-export function utf8ToBase64Ponyfill(
-  text: string,
-  alphabet?: Base64Alphabet,
-): string {
-  const bytes = textEncoder.encode(text)
-  return toString(bytes, alphabet)
-}

package/src/utf8.ts

@@ -1,128 +0,0 @@
-import { Base64Alphabet } from './uint8array.js'
-import {
-  utf8FromBase64Node,
-  utf8FromBase64Ponyfill,
-} from './utf8-from-base64.js'
-import { utf8FromBytesNative, utf8FromBytesNode } from './utf8-from-bytes.js'
-import { graphemeLenNative, graphemeLenPonyfill } from './utf8-grapheme-len.js'
-import { utf8LenCompute, utf8LenNode } from './utf8-len.js'
-import { utf8ToBase64Node, utf8ToBase64Ponyfill } from './utf8-to-base64.js'
-
-/**
- * Converts a Uint8Array to a UTF-8 string.
- *
- * Uses Node.js Buffer when available for performance, falling back to
- * TextDecoder in environments without Buffer support.
- *
- * @param bytes - The binary data to decode
- * @returns The decoded string (as UTF-16 JavaScript string)
- *
- * @example
- * ```typescript
- * import { utf8FromBytes } from '@atproto/lex-data'
- *
- * const bytes = new Uint8Array([72, 101, 108, 108, 111])
- * utf8FromBytes(bytes)  // 'Hello'
- * ```
- */
-export const utf8FromBytes = utf8FromBytesNode ?? utf8FromBytesNative
-
-/**
- * Counts the number of grapheme clusters (user-perceived characters) in a string.
- *
- * Grapheme clusters represent what users typically think of as "characters",
- * handling complex cases like:
- * - Emoji with skin tones and ZWJ sequences (e.g., family emoji)
- * - Combined characters (e.g., 'e' + combining accent)
- * - Regional indicator pairs (flag emoji)
- *
- * Uses native {@link Intl.Segmenter} when available, falling back to a ponyfill.
- *
- * @param str - The string to measure
- * @returns The number of grapheme clusters
- *
- * @example
- * ```typescript
- * import { graphemeLen } from '@atproto/lex-data'
- *
- * graphemeLen('hello')        // 5
- * graphemeLen('cafe\u0301')   // 4 (cafe with combining accent)
- * graphemeLen('\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}')  // 1 (family emoji)
- * ```
- */
-export const graphemeLen: (str: string) => number =
-  /* v8 ignore next -- @preserve */ graphemeLenNative ?? graphemeLenPonyfill
-
-/* v8 ignore next -- @preserve */
-if (graphemeLen === graphemeLenPonyfill) {
-  /*#__PURE__*/
-  console.warn(
-    '[@atproto/lex-data]: Intl.Segmenter is not available in this environment. Falling back to ponyfill implementation.',
-  )
-}
-
-/**
- * Calculates the UTF-8 byte length of a string.
- *
- * Returns the number of bytes the string would occupy when encoded as UTF-8.
- * This is important for Lexicon validation where schemas specify byte limits.
- *
- * Uses Node.js Buffer.byteLength when available for performance,
- * falling back to a computed implementation.
- *
- * @param str - The string to measure
- * @returns The UTF-8 byte length
- *
- * @example
- * ```typescript
- * import { utf8Len } from '@atproto/lex-data'
- *
- * utf8Len('hello')      // 5 (ASCII: 1 byte per char)
- * utf8Len('\u00e9')     // 2 (e with accent: 2 bytes)
- * utf8Len('\u{1F600}')  // 4 (emoji: 4 bytes)
- * utf8Len('\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}')  // 25 (family emoji)
- * ```
- */
-export const utf8Len: (string: string) => number =
-  /* v8 ignore next -- @preserve */ utf8LenNode ?? utf8LenCompute
-
-/**
- * Encodes a UTF-8 string to base64.
- *
- * First encodes the string as UTF-8 bytes, then encodes those bytes as base64.
- *
- * @param str - The string to encode
- * @param alphabet - The base64 alphabet to use ('base64' or 'base64url')
- * @returns The base64-encoded string
- *
- * @example
- * ```typescript
- * import { utf8ToBase64 } from '@atproto/lex-data'
- *
- * utf8ToBase64('Hello')  // 'SGVsbG8='
- * ```
- */
-export const utf8ToBase64: (str: string, alphabet?: Base64Alphabet) => string =
-  /* v8 ignore next -- @preserve */ utf8ToBase64Node ?? utf8ToBase64Ponyfill
-
-/**
- * Decodes a base64 string to UTF-8.
- *
- * Decodes the base64 to bytes, then interprets those bytes as UTF-8 text.
- *
- * @param b64 - The base64 string to decode
- * @param alphabet - The base64 alphabet to use ('base64' or 'base64url')
- * @returns The decoded UTF-8 string
- *
- * @example
- * ```typescript
- * import { utf8FromBase64 } from '@atproto/lex-data'
- *
- * utf8FromBase64('SGVsbG8=')  // 'Hello'
- * ```
- */
-export const utf8FromBase64: (
-  b64: string,
-  alphabet?: Base64Alphabet,
-) => string =
-  /* v8 ignore next -- @preserve */ utf8FromBase64Node ?? utf8FromBase64Ponyfill

package/tsconfig.build.json

@@ -1,12 +0,0 @@
-{
-  "extends": ["../../../tsconfig/isomorphic.json"],
-  "include": ["./src"],
-  "exclude": ["**/*.test.ts"],
-  "compilerOptions": {
-    "noImplicitAny": true,
-    "importHelpers": true,
-    "target": "ES2023",
-    "rootDir": "./src",
-    "outDir": "./dist",
-  },
-}

package/tsconfig.json

@@ -1,7 +0,0 @@
-{
-  "include": [],
-  "references": [
-    { "path": "./tsconfig.build.json" },
-    { "path": "./tsconfig.tests.json" },
-  ],
-}

package/tsconfig.tests.json

@@ -1,8 +0,0 @@
-{
-  "extends": "../../../tsconfig/vitest.json",
-  "include": ["./tests", "./src/**/*.test.ts", "./src/core-js.d.ts"],
-  "compilerOptions": {
-    "noImplicitAny": true,
-    "rootDir": "./",
-  },
-}