@atproto/lex-data 0.0.14 → 0.1.0-next.0

package/src/blob.ts

@@ -1,25 +1,172 @@
-import { Cid, RawCid, ifCid, validateCidString } from './cid.js'
+import {
+  CheckCidOptions,
+  Cid,
+  RawCid,
+  ifCid,
+  parseCid,
+  validateCidString,
+} from './cid.js'
 import { LexValue } from './lex.js'
 import { isPlainObject, isPlainProto } from './object.js'
 
+/**
+ * Options to use with {@link ifCid}, {@link validateCidString}, and related CID
+ * validation functions when validating CIDs in BlobRefs, in strict mode. This
+ * ensures that the CID is a {@link RawCid} (CID v1, raw multicodec, sha256
+ * multihash), which is the expected format for blob references in the AT
+ * Protocol data model.
+ */
+const STRICT_CID_CHECK_OPTIONS: CheckCidOptions = { flavor: 'raw' }
+
 // Number.isSafeInteger is actually safe to use with non-number values, so we
 // can use it as a type guard.
 const isSafeInteger = Number.isSafeInteger as (v: unknown) => v is number
 
+/**
+ * Reference to binary data (like images, videos, etc.) in the AT Protocol data
+ * model.
+ *
+ * This type represents a reference to a blob of binary data, identified by its
+ * content hash (CID) and accompanied by metadata such as MIME type and size.
+ *
+ * The {@link BlobRef} type is a union of the current {@link TypedBlobRef}
+ * format and the legacy {@link LegacyBlobRef} format.
+ */
+export type BlobRef<Ref extends Cid = Cid> = TypedBlobRef<Ref> | LegacyBlobRef
+
+/**
+ * Options for validating a {@link BlobRef}.
+ */
+export type BlobRefCheckOptions = {
+  /**
+   * If `false`, skips strict CID validation of {@link BlobRef.ref}, allowing
+   * any valid CID. Otherwise, validates that the CID is v1, uses the raw
+   * multicodec, and has a sha256 multihash.
+   *
+   * @default true
+   */
+  strict?: boolean
+}
+
+/**
+ * Type guard to check if a value is a valid {@link BlobRef}, which can be
+ * either a {@link TypedBlobRef} or a {@link LegacyBlobRef}. By default, strict
+ * CID validation is applied to ensure that the CID in the blob reference is in
+ * the expected format for the AT Protocol, but this can be relaxed with the
+ * `strict: false` option.
+ */
+export function isBlobRef(input: unknown): input is BlobRef<RawCid>
+export function isBlobRef<TOptions extends BlobRefCheckOptions>(
+  input: unknown,
+  options: TOptions,
+): input is LegacyBlobRef | InferTypedBlobRef<TOptions>
+export function isBlobRef(
+  input: unknown,
+  options?: BlobRefCheckOptions,
+): input is BlobRef<RawCid>
+export function isBlobRef(
+  input: unknown,
+  options?: BlobRefCheckOptions,
+): input is BlobRef {
+  return (input as any)?.$type === 'blob'
+    ? isTypedBlobRef(input, options)
+    : isLegacyBlobRef(input, options)
+}
+
+/**
+ * Extracts the MIME type from a {@link BlobRef}.
+ *
+ * @example
+ * ```ts
+ * const mimeType = getBlobMime(blobRef)
+ * console.log(mimeType)  // e.g., 'image/jpeg'
+ * ```
+ */
+export function getBlobMime(blob: BlobRef): string
+export function getBlobMime(blob?: BlobRef): string | undefined
+export function getBlobMime(blob?: BlobRef): string | undefined {
+  return blob?.mimeType
+}
+
+/**
+ * Extracts the size (in bytes) from a {@link TypedBlobRef}. For
+ * {@link LegacyBlobRef}, size information is not available, so this function
+ * returns `undefined` for legacy refs.
+ *
+ * @note The size property, in blob refs, cannot be 100% trusted since the PDS
+ * might not have a local copy of the blob (to check the size against) and might
+ * just be passing through the blob ref from the client without validating it.
+ * So, while this function can be useful for getting size information when
+ * available, it should not be solely relied upon for critical functionality
+ * without additional validation.
+ *
+ * @example
+ * ```ts
+ * const size = getBlobSize(blobRef)
+ * if (size !== undefined) {
+ *   console.log(`Blob size: ${size} bytes`)
+ * } else {
+ *   console.log('Size information not available for legacy blob ref')
+ * }
+ * ```
+ */
+export function getBlobSize(blob: BlobRef): number | undefined {
+  if ('$type' in blob && blob.size >= 0) return blob.size
+  // LegacyBlobRef doesn't have size information
+  return undefined
+}
+
+/**
+ * Extracts the {@link Cid} from a {@link BlobRef}.
+ *
+ * @throws If the input input is a {@link LegacyBlobRef} with an invalid CID string
+ * @example
+ * ```ts
+ * const cid = getBlobCid(blobRef)
+ * console.log(cid.bytes)
+ * ```
+ */
+export function getBlobCid(blob: BlobRef): Cid
+export function getBlobCid(blob?: BlobRef): Cid | undefined
+export function getBlobCid(blob?: BlobRef): Cid | undefined {
+  if (!blob) return undefined
+  return '$type' in blob ? blob.ref : parseCid(blob.cid)
+}
+
+/**
+ * Extracts the CID string from a {@link BlobRef}.
+ *
+ * This is similar to `getBlobCid(blob).toString()` but is more optimized since
+ * the CID string is already available in the legacy format and we can avoid
+ * parsing it into a CID object just to convert it back to a string.
+ *
+ * @example
+ * ```ts
+ * const cidString = getBlobCidString(blobRef)
+ * console.log(cidString)
+ * ```
+ */
+export function getBlobCidString(blob: BlobRef): string
+export function getBlobCidString(blob?: BlobRef): string | undefined
+export function getBlobCidString(blob?: BlobRef): string | undefined {
+  if (!blob) return undefined
+  return '$type' in blob ? blob.ref.toString() : blob.cid
+}
+
 /**
  * Reference to binary data (like images, videos, etc.) in the AT Protocol data model.
  *
- * A BlobRef is a {@link LexMap} with a specific structure that identifies binary
- * content by its content hash (CID), along with metadata about the content type
- * and size.
+ * A {@link TypedBlobRef} is a {@link LexMap} with a specific structure that
+ * identifies binary content by its content hash (CID), along with metadata
+ * about the content type and size.
  *
  * @typeParam Ref - The type of CID reference, defaults to any {@link Cid}
  *
  * @example
  * ```typescript
- * import type { BlobRef } from '@atproto/lex-data'
+ * import type { TypedBlobRef } from '@atproto/lex-data'
  *
- * const imageRef: BlobRef = {
+ * const imageRef: TypedBlobRef = {
  *   $type: 'blob',
  *   mimeType: 'image/jpeg',
  *   ref: cid,  // CID of the blob content
@@ -27,41 +174,27 @@ const isSafeInteger = Number.isSafeInteger as (v: unknown) => v is number
  * }
  * ```
  *
- * @see {@link isBlobRef} to check if a value is a valid BlobRef
+ * @see {@link isTypedBlobRef} to check if a value is a valid {@link TypedBlobRef}
  * @see {@link LegacyBlobRef} for the older blob reference format
  */
-export type BlobRef<Ref extends Cid = Cid> = {
+export type TypedBlobRef<Ref extends Cid = Cid> = {
   $type: 'blob'
   mimeType: string
   ref: Ref
   size: number
 }
 
-/**
- * Options for validating a {@link BlobRef}.
- */
-export type BlobRefCheckOptions = {
-  /**
-   * If `false`, skips strict CID validation of {@link BlobRef.ref}, allowing
-   * any valid CID. Otherwise, validates that the CID is v1, uses the raw
-   * multicodec, and has a sha256 multihash.
-   *
-   * @default true
-   */
-  strict?: boolean
-}
-
 /**
  * Infers the BlobRef type based on the check options.
  *
  * @typeParam TOptions - The options used for checking
  */
-export type InferCheckedBlobRef<TOptions extends BlobRefCheckOptions> =
+export type InferTypedBlobRef<TOptions extends BlobRefCheckOptions> =
   TOptions extends { strict: false }
-    ? BlobRef
+    ? TypedBlobRef
     : { strict: boolean } extends TOptions
-      ? BlobRef
-      : BlobRef<RawCid>
+      ? TypedBlobRef
+      : TypedBlobRef<RawCid>
 
 /**
  * Type guard to check if a value is a valid {@link BlobRef}.
@@ -78,32 +211,32 @@ export type InferCheckedBlobRef<TOptions extends BlobRefCheckOptions> =
  *
  * @example
  * ```typescript
- * import { isBlobRef } from '@atproto/lex-data'
+ * import { isTypedBlobRef } from '@atproto/lex-data'
  *
- * if (isBlobRef(data)) {
+ * if (isTypedBlobRef(data)) {
  *   console.log(data.mimeType)  // e.g., 'image/jpeg'
  *   console.log(data.size)      // e.g., 12345
  * }
  *
  * // Allow any valid CID (not just raw CIDs)
- * if (isBlobRef(data, { strict: false })) {
+ * if (isTypedBlobRef(data, { strict: false })) {
  *   // ...
  * }
  * ```
  */
-export function isBlobRef(input: unknown): input is BlobRef<RawCid>
-export function isBlobRef<TOptions extends BlobRefCheckOptions>(
+export function isTypedBlobRef(input: unknown): input is TypedBlobRef<RawCid>
+export function isTypedBlobRef<TOptions extends BlobRefCheckOptions>(
   input: unknown,
   options: TOptions,
-): input is InferCheckedBlobRef<TOptions>
-export function isBlobRef(
+): input is InferTypedBlobRef<TOptions>
+export function isTypedBlobRef(
   input: unknown,
   options?: BlobRefCheckOptions,
-): input is BlobRef
-export function isBlobRef(
+): input is TypedBlobRef<RawCid>
+export function isTypedBlobRef(
   input: unknown,
   options?: BlobRefCheckOptions,
-): input is BlobRef {
+): input is TypedBlobRef {
   if (!isPlainObject(input)) {
     return false
   }
@@ -143,7 +276,7 @@ export function isBlobRef(
   const cid = ifCid(
     ref,
     // Strict unless explicitly disabled
-    options?.strict === false ? undefined : { flavor: 'raw' },
+    options?.strict === false ? undefined : STRICT_CID_CHECK_OPTIONS,
   )
   if (!cid) {
     return false
@@ -185,9 +318,6 @@ export type LegacyBlobRef = {
  * - `mimeType` must be a non-empty string
  * - No additional properties allowed
  *
- * @param input - The value to check
- * @returns `true` if the input is a valid LegacyBlobRef
- *
  * @example
  * ```typescript
  * import { isLegacyBlobRef } from '@atproto/lex-data'
@@ -198,9 +328,12 @@ export type LegacyBlobRef = {
  * }
  * ```
  *
- * @see {@link isBlobRef} for checking the current blob reference format
+ * @see {@link isTypedBlobRef} for checking the current blob reference format
  */
-export function isLegacyBlobRef(input: unknown): input is LegacyBlobRef {
+export function isLegacyBlobRef(
+  input: unknown,
+  options?: BlobRefCheckOptions,
+): input is LegacyBlobRef {
   if (!isPlainObject(input)) {
     return false
   }
@@ -220,7 +353,12 @@ export function isLegacyBlobRef(input: unknown): input is LegacyBlobRef {
     }
   }
 
-  if (!validateCidString(cid)) {
+  if (
+    !validateCidString(
+      cid,
+      options?.strict === false ? undefined : STRICT_CID_CHECK_OPTIONS,
+    )
+  ) {
     return false
   }
 
@@ -247,10 +385,10 @@ export type EnumBlobRefsOptions = BlobRefCheckOptions & {
  */
 export type InferEnumBlobRefs<TOptions extends EnumBlobRefsOptions> =
   TOptions extends { allowLegacy: true }
-    ? InferCheckedBlobRef<TOptions> | LegacyBlobRef
+    ? InferTypedBlobRef<TOptions> | LegacyBlobRef
     : { allowLegacy: boolean } extends TOptions
-      ? InferCheckedBlobRef<TOptions> | LegacyBlobRef
-      : InferCheckedBlobRef<TOptions>
+      ? InferTypedBlobRef<TOptions> | LegacyBlobRef
+      : InferTypedBlobRef<TOptions>
 
 /**
  * Generator that enumerates all {@link BlobRef}s (and, optionally,
@@ -280,8 +418,8 @@ export type InferEnumBlobRefs<TOptions extends EnumBlobRefsOptions> =
  * }
  *
  * // Include legacy blob references
- * for (const ref of enumBlobRefs(record, { allowLegacy: true })) {
- *   // ref may be BlobRef or LegacyBlobRef
+ * for (const ref of enumBlobRefs(record, { allowLegacy: true, strict: false })) {
+ *   // ref may be BlobRef or LegacyBlobRef, with relaxed CID validation
  * }
  * ```
  */
@@ -295,11 +433,11 @@ export function enumBlobRefs<TOptions extends EnumBlobRefsOptions>(
 export function enumBlobRefs(
   input: LexValue,
   options?: EnumBlobRefsOptions,
-): Generator<BlobRef | LegacyBlobRef, void, unknown>
+): Generator<BlobRef, void, unknown>
 export function* enumBlobRefs(
   input: LexValue,
   options?: EnumBlobRefsOptions,
-): Generator<BlobRef | LegacyBlobRef, void, unknown> {
+): Generator<BlobRef, void, unknown> {
   // LegacyBlobRef not included by default
   const includeLegacy = options?.allowLegacy === true
 
@@ -322,9 +460,9 @@ export function* enumBlobRefs(
       } else if (isPlainProto(value)) {
         if (visited.has(value)) continue
         visited.add(value)
-        if (isBlobRef(value, options)) {
+        if (isTypedBlobRef(value, options)) {
           yield value
-        } else if (includeLegacy && isLegacyBlobRef(value)) {
+        } else if (includeLegacy && isLegacyBlobRef(value, options)) {
           yield value
         } else {
           for (const v of Object.values(value)) {

package/src/cid-implementation.test.ts

@@ -58,13 +58,13 @@ describe(BytesCid, () => {
   it('throws an error for invalid CID bytes', () => {
     expect(
       () => new BytesCid(new Uint8Array([2, 0x55, 0x12, 3, 1, 2, 3])),
-    ).toThrowError('Unsupported CID version')
-    expect(() => new BytesCid(new Uint8Array([1, 0x55, 0x12]))).toThrowError(
+    ).toThrow('Unsupported CID version')
+    expect(() => new BytesCid(new Uint8Array([1, 0x55, 0x12]))).toThrow(
       'CID bytes are too short',
     )
     expect(
       () => new BytesCid(new Uint8Array([1, 0x55, 0x12, 4, 1, 2, 3])),
-    ).toThrowError('CID bytes length mismatch')
+    ).toThrow('CID bytes length mismatch')
   })
 })
 

package/src/cid.ts

@@ -87,6 +87,7 @@ declare module 'multiformats/cid' {
    * we update or swap out `multiformats`, `@atproto/lex-data` provides its own
    * stable {@link Cid} interface.
    */
+  // eslint-disable-next-line @typescript-eslint/no-empty-object-type
   interface CID {}
 }
 

package/src/core-js.d.ts

@@ -0,0 +1,2 @@
+declare module 'core-js/es/typed-array/from-base64.js'
+declare module 'core-js/es/typed-array/to-base64.js'

package/src/lib/nodejs-buffer.ts

@@ -1,5 +1,10 @@
 type Encoding = 'utf8' | 'base64' | 'base64url'
 
+// Node's buffer module declares this type internally, but referencing it here
+// would couple this file to @types/node. Local copy keeps this module
+// standalone so it compiles in any environment (see tsconfig/isomorphic.json).
+type WithImplicitCoercion<T> = T | { valueOf(): T }
+
 interface NodeJSBuffer<TArrayBuffer extends ArrayBufferLike = ArrayBufferLike>
   extends Uint8Array<TArrayBuffer> {
   byteLength: number
@@ -12,6 +17,11 @@ interface NodeJSBufferConstructor {
     input: Uint8Array | ArrayBuffer | ArrayBufferView,
   ): NodeJSBuffer<ArrayBuffer>
   from(input: string, encoding?: Encoding): NodeJSBuffer<ArrayBuffer>
+  from<TArrayBuffer extends ArrayBufferLike>(
+    arrayBuffer: WithImplicitCoercion<TArrayBuffer>,
+    byteOffset?: number,
+    length?: number,
+  ): Buffer<TArrayBuffer>
   concat(list: readonly Uint8Array[], totalLength?: number): NodeJSBuffer
   byteLength(input: string, encoding?: Encoding): number
   prototype: NodeJSBuffer

package/src/uint8array-concat.ts

@@ -3,12 +3,16 @@ import { NodeJSBuffer } from './lib/nodejs-buffer.js'
 const Buffer = NodeJSBuffer
 
 export const ui8ConcatNode = Buffer
-  ? function ui8ConcatNode(array: readonly Uint8Array[]): Uint8Array {
-      return Buffer.concat(array)
+  ? function ui8ConcatNode(
+      array: readonly Uint8Array[],
+    ): Uint8Array<ArrayBuffer> {
+      return Buffer.concat(array) as Uint8Array<ArrayBuffer>
     }
   : /* v8 ignore next -- @preserve */ null
 
-export function ui8ConcatPonyfill(array: readonly Uint8Array[]): Uint8Array {
+export function ui8ConcatPonyfill(
+  array: readonly Uint8Array[],
+): Uint8Array<ArrayBuffer> {
   let totalLength = 0
   for (const arr of array) totalLength += arr.length
   const result = new Uint8Array(totalLength)

package/src/uint8array-from-base64.test.ts

@@ -1,5 +1,5 @@
-import 'core-js/modules/es.uint8-array.from-base64.js'
-import 'core-js/modules/es.uint8-array.to-base64.js'
+import 'core-js/es/typed-array/from-base64.js'
+import 'core-js/es/typed-array/to-base64.js'
 
 import { assert, describe, expect, it } from 'vitest'
 import {

package/src/uint8array-to-base64.test.ts

@@ -1,5 +1,5 @@
-import 'core-js/modules/es.uint8-array.from-base64.js'
-import 'core-js/modules/es.uint8-array.to-base64.js'
+import 'core-js/es/typed-array/from-base64.js'
+import 'core-js/es/typed-array/to-base64.js'
 import { assert, describe, expect, it } from 'vitest'
 import {
   toBase64Native,

package/src/uint8array.test.ts

@@ -1,5 +1,5 @@
-import 'core-js/modules/es.uint8-array.from-base64.js'
-import 'core-js/modules/es.uint8-array.to-base64.js'
+import 'core-js/es/typed-array/from-base64.js'
+import 'core-js/es/typed-array/to-base64.js'
 
 import { describe, expect, it } from 'vitest'
 import {

package/src/utf8-from-bytes.test.ts

@@ -0,0 +1,43 @@
+import { assert, describe, expect, it } from 'vitest'
+import { utf8FromBytesNative, utf8FromBytesNode } from './utf8-from-bytes.js'
+
+for (const utf8FromBytes of [utf8FromBytesNode, utf8FromBytesNative] as const) {
+  assert(utf8FromBytes, 'utf8FromBytes implementation should not be null')
+  describe(utf8FromBytes, () => {
+    it('decodes empty Uint8Array', () => {
+      const decoded = utf8FromBytes(new Uint8Array(0))
+      expect(typeof decoded).toBe('string')
+      expect(decoded).toBe('')
+    })
+
+    it('decodes 10MB', () => {
+      const bytes = Buffer.allocUnsafe(10_000_000).fill('🐩')
+      const decoded = utf8FromBytes(bytes)
+      expect(decoded).toBe('🐩'.repeat(10_000_000 / 4))
+    })
+
+    for (const string of [
+      '',
+      '\0\0',
+      '\0\0\0',
+      '\0\0\0\0',
+      '__',
+      'é',
+      'àç',
+      '\0éàç',
+      '```\x1b',
+      'aaa',
+      'Hello, World!',
+      '😀😃😄😁😆😅😂🤣😊😇',
+      '👩‍💻👨‍💻👩‍🔬👨‍🔬👩‍🚀👨‍🚀',
+      '🌍🌎🌏🌐🪐🌟✨⚡🔥💧',
+    ] as const) {
+      const buffer = Buffer.from(string, 'utf8')
+
+      it(`decodes ${JSON.stringify(string)}`, () => {
+        const decoded = utf8FromBytes(buffer)
+        expect(decoded).toBe(string)
+      })
+    }
+  })
+}

package/src/utf8-from-bytes.ts

@@ -0,0 +1,21 @@
+import { NodeJSBuffer } from './lib/nodejs-buffer.js'
+
+const Buffer = NodeJSBuffer
+
+export const utf8FromBytesNode = Buffer
+  ? function utf8FromBytesNode(bytes: Uint8Array): string {
+      // @NOTE Buffer.from(bytes) creates a copy of the ArrayBuffer. The following
+      // allows us to avoid the copy by creating a Buffer that shares the same
+      // memory as the input Uint8Array.
+      const buffer = Buffer.from(
+        bytes.buffer,
+        bytes.byteOffset,
+        bytes.byteLength,
+      )
+      return buffer.toString('utf8')
+    }
+  : /* v8 ignore next -- @preserve */ null
+
+export function utf8FromBytesNative(bytes: Uint8Array): string {
+  return new TextDecoder('utf-8').decode(bytes)
+}

package/src/utf8.ts

@@ -3,10 +3,30 @@ import {
   utf8FromBase64Node,
   utf8FromBase64Ponyfill,
 } from './utf8-from-base64.js'
+import { utf8FromBytesNative, utf8FromBytesNode } from './utf8-from-bytes.js'
 import { graphemeLenNative, graphemeLenPonyfill } from './utf8-grapheme-len.js'
 import { utf8LenCompute, utf8LenNode } from './utf8-len.js'
 import { utf8ToBase64Node, utf8ToBase64Ponyfill } from './utf8-to-base64.js'
 
+/**
+ * Converts a Uint8Array to a UTF-8 string.
+ *
+ * Uses Node.js Buffer when available for performance, falling back to
+ * TextDecoder in environments without Buffer support.
+ *
+ * @param bytes - The binary data to decode
+ * @returns The decoded string (as UTF-16 JavaScript string)
+ *
+ * @example
+ * ```typescript
+ * import { utf8FromBytes } from '@atproto/lex-data'
+ *
+ * const bytes = new Uint8Array([72, 101, 108, 108, 111])
+ * utf8FromBytes(bytes)  // 'Hello'
+ * ```
+ */
+export const utf8FromBytes = utf8FromBytesNode ?? utf8FromBytesNative
+
 /**
  * Counts the number of grapheme clusters (user-perceived characters) in a string.
  *

package/tsconfig.tests.json

@@ -1,6 +1,6 @@
 {
   "extends": "../../../tsconfig/vitest.json",
-  "include": ["./tests", "./src/**/*.test.ts"],
+  "include": ["./tests", "./src/**/*.test.ts", "./src/core-js.d.ts"],
   "compilerOptions": {
     "noImplicitAny": true,
     "rootDir": "./",