@atproto/lex-data 0.0.5 → 0.0.7

package/src/blob.ts

@@ -1,25 +1,47 @@
-import {
-  Cid,
-  RAW_BIN_MULTICODEC,
-  SHA2_256_MULTIHASH_CODE,
-  asCid,
-  parseCid,
-} from './cid.js'
-import { isPlainObject } from './object.js'
+import { Cid, RawCid, ifCid, validateCidString } from './cid.js'
+import { LexValue } from './lex.js'
+import { isPlainObject, isPlainProto } from './object.js'
 
 /**
  * @note {@link BlobRef} is just a {@link LexMap} with a specific shape.
  */
-export type BlobRef = {
+export type BlobRef<Ref extends Cid = Cid> = {
   $type: 'blob'
   mimeType: string
-  ref: Cid
+  ref: Ref
   size: number
 }
 
+export type BlobRefCheckOptions = {
+  /**
+   * If `false`, skips strict CID validation of {@link BlobRef.ref}, allowing
+   * any valid CID. Otherwise, validates that the CID is v1, uses the raw
+   * multicodec, and has a sha256 multihash.
+   *
+   * @defaults to `true`
+   */
+  strict?: boolean
+}
+
+export type InferCheckedBlobRef<TOptions extends BlobRefCheckOptions> =
+  TOptions extends { strict: false }
+    ? BlobRef
+    : { strict: boolean } extends TOptions
+      ? BlobRef
+      : BlobRef<RawCid>
+
+export function isBlobRef(input: unknown): input is BlobRef<RawCid>
+export function isBlobRef<TOptions extends BlobRefCheckOptions>(
+  input: unknown,
+  options: TOptions,
+): input is InferCheckedBlobRef<TOptions>
+export function isBlobRef(
+  input: unknown,
+  options?: BlobRefCheckOptions,
+): input is BlobRef
 export function isBlobRef(
   input: unknown,
-  options?: { strict?: boolean },
+  options?: BlobRefCheckOptions,
 ): input is BlobRef {
   if (!isPlainObject(input)) {
     return false
@@ -54,23 +76,15 @@ export function isBlobRef(
     }
   }
 
-  const cid = asCid(ref)
+  const cid = ifCid(
+    ref,
+    // Strict unless explicitly disabled
+    options?.strict === false ? undefined : { flavor: 'raw' },
+  )
   if (!cid) {
     return false
   }
 
-  if (options?.strict) {
-    if (cid.version !== 1) {
-      return false
-    }
-    if (cid.code !== RAW_BIN_MULTICODEC) {
-      return false
-    }
-    if (cid.multihash.code !== SHA2_256_MULTIHASH_CODE) {
-      return false
-    }
-  }
-
   return true
 }
 
@@ -102,11 +116,81 @@ export function isLegacyBlobRef(input: unknown): input is LegacyBlobRef {
     }
   }
 
-  try {
-    parseCid(cid)
-  } catch {
+  if (!validateCidString(cid)) {
     return false
   }
 
   return true
 }
+
+export type EnumBlobRefsOptions = BlobRefCheckOptions & {
+  /**
+   * @defaults to `false`
+   */
+  allowLegacy?: boolean
+}
+
+export type InferEnumBlobRefs<TOptions extends EnumBlobRefsOptions> =
+  TOptions extends { allowLegacy: true }
+    ? InferCheckedBlobRef<TOptions> | LegacyBlobRef
+    : { allowLegacy: boolean } extends TOptions
+      ? InferCheckedBlobRef<TOptions> | LegacyBlobRef
+      : InferCheckedBlobRef<TOptions>
+
+/**
+ * Enumerates all {@link BlobRef}s (and, optionally, {@link LegacyBlobRef}s)
+ * found within a {@link LexValue}.
+ */
+export function enumBlobRefs(
+  input: LexValue,
+): Generator<BlobRef<RawCid>, void, unknown>
+export function enumBlobRefs<TOptions extends EnumBlobRefsOptions>(
+  input: LexValue,
+  options: TOptions,
+): Generator<InferEnumBlobRefs<TOptions>, void, unknown>
+export function enumBlobRefs(
+  input: LexValue,
+  options?: EnumBlobRefsOptions,
+): Generator<BlobRef | LegacyBlobRef, void, unknown>
+export function* enumBlobRefs(
+  input: LexValue,
+  options?: EnumBlobRefsOptions,
+): Generator<BlobRef | LegacyBlobRef, void, unknown> {
+  // LegacyBlobRef not included by default
+  const includeLegacy = options?.allowLegacy === true
+
+  // Using a stack to avoid recursion depth issues.
+  const stack: LexValue[] = [input]
+
+  // Since we are using a stack, we could end-up in an infinite loop with cyclic
+  // structures. Cyclic structures are not valid LexValues and should, thus,
+  // never occur, but let's be safe.
+  const visited = new Set<object>()
+
+  do {
+    const value = stack.pop()!
+
+    if (value != null && typeof value === 'object') {
+      if (Array.isArray(value)) {
+        if (visited.has(value)) continue
+        visited.add(value)
+        stack.push(...value)
+      } else if (isPlainProto(value)) {
+        if (visited.has(value)) continue
+        visited.add(value)
+        if (isBlobRef(value, options)) {
+          yield value
+        } else if (includeLegacy && isLegacyBlobRef(value)) {
+          yield value
+        } else {
+          for (const v of Object.values(value)) {
+            if (v != null) stack.push(v)
+          }
+        }
+      }
+    }
+  } while (stack.length > 0)
+
+  // Optimization: ease GC's work
+  visited.clear()
+}

package/src/cid.test.ts

@@ -2,8 +2,8 @@ import { CID } from 'multiformats/cid'
 import { sha256, sha512 } from 'multiformats/hashes/sha2'
 import { describe, expect, it } from 'vitest'
 import {
-  RAW_BIN_MULTICODEC,
-  createCid,
+  DAG_CBOR_MULTICODEC,
+  RAW_MULTICODEC,
   decodeCid,
   ensureValidCidString,
   isCid,
@@ -23,7 +23,7 @@ describe(isCid, () => {
     it('returns true for CID v0 and v1', async () => {
       const digest = await sha256.digest(Buffer.from('hello world'))
       const cidV0 = CID.createV0(digest)
-      const cidV1 = CID.createV1(RAW_BIN_MULTICODEC, digest)
+      const cidV1 = CID.createV1(RAW_MULTICODEC, digest)
       expect(isCid(cidV0)).toBe(true)
       expect(isCid(cidV1)).toBe(true)
     })
@@ -35,41 +35,58 @@ describe(isCid, () => {
     })
   })
 
-  describe('strict mode', () => {
-    it('returns true for valid CIDs in strict mode', async () => {
-      const digest = await sha256.digest(Buffer.from('hello world'))
-      const cid = CID.createV1(RAW_BIN_MULTICODEC, digest)
-      expect(isCid(cid, { strict: true })).toBe(true)
-    })
+  describe('flavors', () => {
+    describe('raw', () => {
+      it('validated "raw" cids', async () => {
+        const digest = await sha256.digest(Buffer.from('hello world'))
+        const cid = CID.createV1(RAW_MULTICODEC, digest)
+        expect(isCid(cid, { flavor: 'raw' })).toBe(true)
+      })
 
-    it('rejects CID v0 when strict option is set', async () => {
-      const digest = await sha256.digest(Buffer.from('hello world'))
-      const cid = CID.createV0(digest)
-      expect(isCid(cid, { strict: true })).toBe(false)
-    })
+      it('allows other hash algorithms', async () => {
+        const digest = await sha512.digest(Buffer.from('hello world'))
+        const cid = CID.createV1(RAW_MULTICODEC, digest)
+        expect(isCid(cid, { flavor: 'raw' })).toBe(true)
+      })
 
-    it('rejects CIDs with invalid hash algorithm', async () => {
-      const digest = await sha512.digest(Buffer.from('hello world'))
-      const cid = CID.createV1(RAW_BIN_MULTICODEC, digest)
-      expect(isCid(cid, { strict: true })).toBe(false)
-    })
+      it('rejects CID v0 when strict option is set', async () => {
+        const digest = await sha256.digest(Buffer.from('hello world'))
+        const cid = CID.createV0(digest)
+        expect(isCid(cid, { flavor: 'raw' })).toBe(false)
+      })
 
-    it('rejects CIDs with invalid code', async () => {
-      const digest = await sha256.digest(Buffer.from('hello world'))
-      const cid = CID.createV1(3333, digest)
-      expect(isCid(cid, { strict: true })).toBe(false)
+      it('rejects CIDs with invalid code', async () => {
+        const digest = await sha256.digest(Buffer.from('hello world'))
+        const cid = CID.createV1(3333, digest)
+        expect(isCid(cid, { flavor: 'raw' })).toBe(false)
+      })
     })
-  })
-})
 
-describe(createCid, () => {
-  it('creates a valid CID v1', async () => {
-    const digest = await sha256.digest(Buffer.from('hello world'))
-    const cid = createCid(RAW_BIN_MULTICODEC, digest)
-    expect(cid.version).toBe(1)
-    expect(cid.code).toBe(RAW_BIN_MULTICODEC)
-    expect(cid.multihash.code).toBe(sha256.code)
-    expect(cid.multihash.digest).toEqual(digest.digest)
+    describe('cbor', () => {
+      it('validated "cbor" cids', async () => {
+        const digest = await sha256.digest(Buffer.from('hello world'))
+        const cid = CID.createV1(DAG_CBOR_MULTICODEC, digest)
+        expect(isCid(cid, { flavor: 'cbor' })).toBe(true)
+      })
+
+      it('rejects CIDs with invalid hash algorithm', async () => {
+        const digest = await sha512.digest(Buffer.from('hello world'))
+        const cid = CID.createV1(RAW_MULTICODEC, digest)
+        expect(isCid(cid, { flavor: 'cbor' })).toBe(false)
+      })
+
+      it('rejects CID v0 when strict option is set', async () => {
+        const digest = await sha256.digest(Buffer.from('hello world'))
+        const cid = CID.createV0(digest)
+        expect(isCid(cid, { flavor: 'cbor' })).toBe(false)
+      })
+
+      it('rejects CIDs with invalid code', async () => {
+        const digest = await sha256.digest(Buffer.from('hello world'))
+        const cid = CID.createV1(3333, digest)
+        expect(isCid(cid, { flavor: 'cbor' })).toBe(false)
+      })
+    })
   })
 })
 

package/src/cid.ts

@@ -1,9 +1,18 @@
 import { CID } from 'multiformats/cid'
+import {
+  create as createDigest,
+  equals as digestEquals,
+} from 'multiformats/hashes/digest'
+import { sha256, sha512 } from 'multiformats/hashes/sha2'
 
-export const DAG_CBOR_MULTICODEC = 0x71
-export const RAW_BIN_MULTICODEC = 0x55
+export const DAG_CBOR_MULTICODEC = 0x71 // DRISL conformant DAG-CBOR
+export type DAG_CBOR_MULTICODEC = typeof DAG_CBOR_MULTICODEC
 
-export const SHA2_256_MULTIHASH_CODE = 0x12
+export const RAW_MULTICODEC = 0x55 // raw binary codec used in DASL CIDs
+export type RAW_MULTICODEC = typeof RAW_MULTICODEC
+
+export const SHA256_MULTIHASH = sha256.code
+export type SHA256_MULTIHASH = typeof SHA256_MULTIHASH
 
 export type MultihashDigest<Code extends number = number> = {
   code: Code
@@ -15,8 +24,8 @@ export type MultihashDigest<Code extends number = number> = {
 declare module 'multiformats/cid' {
   /**
    * @deprecated use the {@link Cid} interface from `@atproto/lex-data`, and
-   * related helpers ({@link asCid}, {@link parseCid}, {@link decodeCid},
-   * {@link createCid}, {@link isCid}), instead.
+   * related helpers ({@link isCid}, {@link ifCid}, {@link asCid},
+   * {@link parseCid}, {@link decodeCid}), instead.
    *
    * This is marked as deprecated because we want to discourage direct usage of
    * `multiformats/cid` in dependent packages, and instead have them rely on the
@@ -69,60 +78,216 @@ export interface Cid {
   toString(): string
 }
 
-export function asCid(value: unknown): Cid | null {
-  return CID.asCID(value)
+/**
+ * Represents the cid of raw binary data (like media blobs).
+ * @see {@link https://atproto.com/specs/data-model#link-and-cid-formats ATproto Data Model - Link and CID Formats}
+ */
+export interface RawCid extends Cid {
+  version: 1
+  code: RAW_MULTICODEC
+}
+
+export function isRawCid(cid: Cid): cid is RawCid {
+  return cid.version === 1 && cid.code === RAW_MULTICODEC
+}
+
+/**
+ * Represents a DASL compliant CID.
+ * @see {@link https://dasl.ing/cid.html DASL-CIDs}
+ */
+export interface DaslCid extends Cid {
+  version: 1
+  code: RAW_MULTICODEC | DAG_CBOR_MULTICODEC
+  multihash: MultihashDigest<SHA256_MULTIHASH>
 }
 
-export function parseCid(input: string): Cid {
-  return CID.parse(input)
+export function isDaslCid(cid: Cid): cid is DaslCid {
+  return (
+    cid.version === 1 &&
+    (cid.code === RAW_MULTICODEC || cid.code === DAG_CBOR_MULTICODEC) &&
+    cid.multihash.code === SHA256_MULTIHASH &&
+    cid.multihash.size === 32 // Should always be 32 bytes (256 bits) for SHA-256
+  )
 }
 
-export function decodeCid(bytes: Uint8Array): Cid {
-  return CID.decode(bytes)
+/**
+ * Represents the cid of ATProto DAG-CBOR data (like repository MST nodes).
+ * @see {@link https://atproto.com/specs/data-model#link-and-cid-formats ATproto Data Model - Link and CID Formats}
+ */
+export interface CborCid extends DaslCid {
+  code: DAG_CBOR_MULTICODEC
 }
 
-export function createCid(code: number, digest: MultihashDigest): Cid {
-  return CID.createV1(code, digest)
+export function isCborCid(cid: Cid): cid is CborCid {
+  return cid.code === DAG_CBOR_MULTICODEC && isDaslCid(cid)
 }
 
-export function isCid(
+export type CidCheckOptions = {
+  flavor?: 'raw' | 'cbor' | 'dasl'
+}
+export type InferCheckedCid<TOptions> = TOptions extends { flavor: 'raw' }
+  ? RawCid
+  : TOptions extends { flavor: 'cbor' }
+    ? CborCid
+    : Cid
+
+/**
+ * Coerces the input value to a Cid, or returns null if not possible.
+ */
+export function ifCid<TOptions extends CidCheckOptions>(
   value: unknown,
-  options?: { strict?: boolean },
-): value is Cid {
-  const cid = asCid(value)
+  options: TOptions,
+): InferCheckedCid<TOptions> | null
+export function ifCid(value: unknown, options?: CidCheckOptions): Cid | null
+export function ifCid(value: unknown, options?: CidCheckOptions): Cid | null {
+  const cid = CID.asCID(value)
   if (!cid) {
-    return false
+    return null
   }
 
-  if (options?.strict) {
-    if (cid.version !== 1) {
-      return false
-    }
-    if (cid.code !== RAW_BIN_MULTICODEC && cid.code !== DAG_CBOR_MULTICODEC) {
-      return false
-    }
-    if (cid.multihash.code !== SHA2_256_MULTIHASH_CODE) {
-      return false
-    }
+  switch (options?.flavor) {
+    case 'cbor':
+      return isCborCid(cid) ? cid : null
+    case 'raw':
+      return isRawCid(cid) ? cid : null
+    case 'dasl':
+      return isDaslCid(cid) ? cid : null
+    default:
+      return cid
   }
+}
 
-  return true
+export function isCid<TOptions extends CidCheckOptions>(
+  value: unknown,
+  options: TOptions,
+): value is InferCheckedCid<TOptions>
+export function isCid(value: unknown, options?: CidCheckOptions): value is Cid
+export function isCid(value: unknown, options?: CidCheckOptions): value is Cid {
+  return ifCid(value, options) !== null
 }
 
-export function validateCidString(input: string): boolean {
-  return parseCidString(input)?.toString() === input
+/**
+ * Coerces the input value to a Cid, or throws if not possible.
+ */
+export function asCid<TOptions extends CidCheckOptions>(
+  value: unknown,
+  options: TOptions,
+): InferCheckedCid<TOptions>
+export function asCid(value: unknown, options?: CidCheckOptions): Cid
+export function asCid(value: unknown, options?: CidCheckOptions): Cid {
+  const cid = ifCid(value, options)
+  if (cid) return cid
+  throw new Error('Not a valid CID')
 }
 
-export function parseCidString(input: string): Cid | undefined {
+/**
+ * Parses a CID string into a Cid object.
+ *
+ * @throws if the input is not a valid CID string.
+ */
+export function parseCid<TOptions extends CidCheckOptions>(
+  input: string,
+  options: TOptions,
+): InferCheckedCid<TOptions>
+export function parseCid(input: string, options?: CidCheckOptions): Cid
+export function parseCid(input: string, options?: CidCheckOptions): Cid {
+  const cid = CID.parse(input)
+  return asCid(cid, options)
+}
+
+/**
+ * Decodes a CID from its binary representation.
+ *
+ * @see {@link https://dasl.ing/cid.html DASL-CIDs}
+ * @throws if the input do not represent a valid DASL {@link Cid}
+ */
+export function decodeCid<TOptions extends CidCheckOptions>(
+  cidBytes: Uint8Array,
+  options: TOptions,
+): InferCheckedCid<TOptions>
+export function decodeCid(cidBytes: Uint8Array, options?: CidCheckOptions): Cid
+export function decodeCid(
+  cidBytes: Uint8Array,
+  options?: CidCheckOptions,
+): Cid {
+  const cid = CID.decode(cidBytes)
+  return asCid(cid, options)
+}
+
+export function validateCidString(
+  input: string,
+  options?: CidCheckOptions,
+): boolean {
+  return parseCidString(input, options)?.toString() === input
+}
+
+export function parseCidString<TOptions extends CidCheckOptions>(
+  input: string,
+  options: TOptions,
+): InferCheckedCid<TOptions> | undefined
+export function parseCidString(
+  input: string,
+  options?: CidCheckOptions,
+): Cid | undefined
+export function parseCidString(
+  input: string,
+  options?: CidCheckOptions,
+): Cid | undefined {
   try {
-    return parseCid(input)
+    return parseCid(input, options)
   } catch {
     return undefined
   }
 }
 
-export function ensureValidCidString(input: string): void {
-  if (!validateCidString(input)) {
+export function ensureValidCidString(
+  input: string,
+  options?: CidCheckOptions,
+): void {
+  if (!validateCidString(input, options)) {
     throw new Error(`Invalid CID string`)
   }
 }
+
+/**
+ * Verifies whether the multihash of a given {@link cid} matches the hash of the provided {@link bytes}.
+ * @params cid The CID to match against the bytes.
+ * @params bytes The bytes to verify.
+ * @returns true if the CID matches the bytes, false otherwise.
+ */
+export async function isCidForBytes(
+  cid: Cid,
+  bytes: Uint8Array,
+): Promise<boolean> {
+  if (cid.multihash.code === sha256.code) {
+    const digest = await sha256.digest(bytes)
+    return digestEquals(cid.multihash, digest)
+  }
+
+  if (cid.multihash.code === sha512.code) {
+    const digest = await sha512.digest(bytes)
+    return digestEquals(cid.multihash, digest)
+  }
+
+  // Don't know how to verify other multihash codes
+  throw new Error('Unsupported CID multihash')
+}
+
+export async function cidForCbor(bytes: Uint8Array): Promise<CborCid> {
+  const digest = await sha256.digest(bytes)
+  return CID.createV1(DAG_CBOR_MULTICODEC, digest) as CborCid
+}
+
+export async function cidForRawBytes(bytes: Uint8Array): Promise<RawCid> {
+  const digest = await sha256.digest(bytes)
+  return CID.createV1(RAW_MULTICODEC, digest) as RawCid
+}
+
+export function cidForRawHash(hash: Uint8Array): RawCid {
+  // Fool-proofing
+  if (hash.length !== 32) {
+    throw new Error(`Invalid SHA-256 hash length: ${hash.length}`)
+  }
+  const digest = createDigest(sha256.code, hash)
+  return CID.createV1(RAW_MULTICODEC, digest) as RawCid
+}

package/src/lex-equals.ts

@@ -1,4 +1,4 @@
-import { asCid, isCid } from './cid.js'
+import { ifCid, isCid } from './cid.js'
 import { LexValue } from './lex.js'
 import { isPlainObject } from './object.js'
 import { ui8Equals } from './uint8array.js'
@@ -44,7 +44,7 @@ export function lexEquals(a: LexValue, b: LexValue): boolean {
   if (isCid(a)) {
     // @NOTE CID.equals returns its argument when it is falsy (e.g. null or
     // undefined) so we need to explicitly check that the output is "true".
-    return asCid(a)!.equals(asCid(b)) === true
+    return ifCid(b)?.equals(a) === true
   } else if (isCid(b)) {
     return false
   }

package/src/uint8array-from-base64.ts

@@ -31,7 +31,7 @@ export const fromBase64Native =
           lastChunkHandling: 'loose',
         })
       }
-    : null
+    : /* v8 ignore next -- @preserve */ null
 
 export const fromBase64Node = Buffer
   ? function fromBase64Node(

package/src/uint8array-to-base64.test.ts

@@ -14,9 +14,9 @@ for (const toBase64 of [
 ] as const) {
   // Tests should run in NodeJS where implementations are either available or
   // polyfilled (see core-js imports above).
-  assert(toBase64 !== null, 'toBase64 implementation should not be null')
+  assert(toBase64, 'toBase64 implementation should not be null')
 
-  describe(toBase64.name, () => {
+  describe(toBase64, () => {
     describe('basic encoding', () => {
       it('encodes empty Uint8Array', () => {
         const encoded = toBase64(new Uint8Array(0))

package/src/uint8array-to-base64.ts

@@ -25,7 +25,7 @@ export const toBase64Native =
       ): string {
         return bytes.toBase64!({ alphabet, omitPadding: true })
       }
-    : null
+    : /* v8 ignore next -- @preserve */ null
 
 export const toBase64Node = Buffer
   ? function toBase64Node(
@@ -45,7 +45,7 @@ export const toBase64Node = Buffer
           : b64.slice(0, -1) // '='
         : b64
     }
-  : null
+  : /* v8 ignore next -- @preserve */ null
 
 export function toBase64Ponyfill(
   bytes: Uint8Array,

package/src/utf8-from-base64.test.ts

@@ -0,0 +1,39 @@
+import { assert, describe, expect, it } from 'vitest'
+import {
+  utf8FromBase64Node,
+  utf8FromBase64Ponyfill,
+} from './utf8-from-base64.js'
+
+const strings = [
+  'Hello, World!',
+  '¡Hola, Mundo!',
+  'こんにちは世界',
+  '😀👩‍💻🌍',
+  '',
+  '𓀀𓁐𓂀𓃰𓄿𓅱𓆑𓇋𓈖𓉔𓊃𓋴𓌳𓍿𓎛𓏏',
+]
+
+for (const utf8FromBase64 of [
+  utf8FromBase64Node,
+  utf8FromBase64Ponyfill,
+] as const) {
+  assert(utf8FromBase64, 'implementation should not be null')
+
+  describe(utf8FromBase64, () => {
+    it('decodes base64 to utf8 string', () => {
+      for (const text of strings) {
+        const b64 = Buffer.from(text, 'utf8').toString('base64')
+        const decoded = utf8FromBase64(b64, 'base64')
+        expect(decoded).toBe(text)
+      }
+    })
+
+    it('decodes base64url to utf8 string', () => {
+      for (const text of strings) {
+        const b64u = Buffer.from(text, 'utf8').toString('base64url')
+        const decoded = utf8FromBase64(b64u, 'base64url')
+        expect(decoded).toBe(text)
+      }
+    })
+  })
+}

package/src/utf8-from-base64.ts

@@ -0,0 +1,23 @@
+import { fromString } from 'uint8arrays/from-string'
+import { NodeJSBuffer } from './lib/nodejs-buffer.js'
+import { Base64Alphabet } from './uint8array-base64.js'
+
+const Buffer = NodeJSBuffer
+
+export const utf8FromBase64Node = Buffer
+  ? function utf8FromBase64Node(
+      b64: string,
+      alphabet: Base64Alphabet = 'base64',
+    ): string {
+      return Buffer.from(b64, alphabet).toString('utf8')
+    }
+  : /* v8 ignore next -- @preserve */ null
+
+const textDecoder = /*#__PURE__*/ new TextDecoder()
+export function utf8FromBase64Ponyfill(
+  b64: string,
+  alphabet?: Base64Alphabet,
+): string {
+  const bytes = fromString(b64, alphabet)
+  return textDecoder.decode(bytes)
+}