@atproto/lex-builder 0.1.4 → 0.1.5

package/src/lexicon-directory-indexer.ts

@@ -1,65 +0,0 @@
-import { readFile, readdir } from 'node:fs/promises'
-import { join } from 'node:path'
-import {
-  LexiconDocument,
-  LexiconIterableIndexer,
-  lexiconDocumentSchema,
-} from '@atproto/lex-document'
-
-/**
- * Options for the {@link LexiconDirectoryIndexer}.
- *
- * @see {@link ReadLexiconsOptions} for available options
- */
-export type LexiconDirectoryIndexerOptions = ReadLexiconsOptions
-
-/**
- * Indexes lexicon documents from a filesystem directory.
- *
- * This class recursively scans a directory for JSON files, parses them as
- * lexicon documents, and provides an iterable interface for processing them.
- * It extends {@link LexiconIterableIndexer} to support both iteration and
- * lookup by NSID.
- */
-export class LexiconDirectoryIndexer extends LexiconIterableIndexer {
-  constructor(options: LexiconDirectoryIndexerOptions) {
-    super(readLexicons(options))
-  }
-}
-
-type ReadLexiconsOptions = {
-  lexicons: string
-  ignoreInvalidLexicons?: boolean
-}
-
-async function* readLexicons(
-  options: ReadLexiconsOptions,
-): AsyncGenerator<LexiconDocument, void, unknown> {
-  for await (const filePath of listFiles(options.lexicons)) {
-    if (filePath.endsWith('.json')) {
-      try {
-        const data = await readFile(filePath, 'utf8')
-        yield lexiconDocumentSchema.parse(JSON.parse(data))
-      } catch (cause) {
-        const message = `Error parsing lexicon document ${filePath}`
-        if (options.ignoreInvalidLexicons) console.error(`${message}:`, cause)
-        else throw new Error(message, { cause })
-      }
-    }
-  }
-}
-
-async function* listFiles(dir: string): AsyncGenerator<string> {
-  const dirents = await readdir(dir, { withFileTypes: true }).catch((err) => {
-    if ((err as any)?.code === 'ENOENT') return []
-    throw err
-  })
-  for (const dirent of dirents) {
-    const res = join(dir, dirent.name)
-    if (dirent.isDirectory()) {
-      yield* listFiles(res)
-    } else if (dirent.isFile() || dirent.isSymbolicLink()) {
-      yield res
-    }
-  }
-}

package/src/polyfill.ts

@@ -1,7 +0,0 @@
-// Node <18.18, 19.x, <20.4 and 21.x do not have these symbols defined
-
-// @ts-expect-error
-Symbol.asyncDispose ??= Symbol.for('nodejs.asyncDispose')
-
-// @ts-expect-error
-Symbol.dispose ??= Symbol.for('nodejs.dispose')

package/src/ref-resolver.test.ts

@@ -1,75 +0,0 @@
-import { Project } from 'ts-morph'
-import { describe, expect, it } from 'vitest'
-import { LexiconDocument, LexiconIndexer } from '@atproto/lex-document'
-import { RefResolver } from './ref-resolver.js'
-
-class DummyIndexer implements LexiconIndexer, AsyncIterable<LexiconDocument> {
-  readonly docs: Map<string, LexiconDocument>
-
-  constructor(docs: LexiconDocument[]) {
-    this.docs = new Map(docs.map((doc) => [doc.id, doc]))
-  }
-
-  async get(id: string): Promise<LexiconDocument> {
-    const doc = this.docs.get(id)
-    if (!doc) {
-      throw new Error(`Document not found: ${id}`)
-    }
-    return doc
-  }
-
-  async *[Symbol.asyncIterator]() {
-    for (const doc of this.docs.values()) {
-      yield doc
-    }
-  }
-}
-
-describe('RefResolver', () => {
-  const docs: LexiconDocument[] = [
-    {
-      lexicon: 1,
-      id: 'com.example.foo',
-      defs: {
-        main: { type: 'token' },
-      },
-    },
-    {
-      lexicon: 1,
-      id: 'com.example.bar',
-      defs: {
-        main: { type: 'token' },
-      },
-    },
-  ]
-
-  it('uses default relative path for external refs', async () => {
-    const project = new Project({ useInMemoryFileSystem: true })
-    const file = project.createSourceFile('/com/example/foo.defs.ts')
-    const indexer = new DummyIndexer(docs)
-    const resolver = new RefResolver(docs[0], file, indexer, {})
-
-    await resolver.resolve('com.example.bar#main')
-
-    const imports = file.getImportDeclarations()
-    expect(imports).toHaveLength(1)
-    expect(imports[0].getModuleSpecifierValue()).toBe('./bar.defs.js')
-  })
-
-  it('uses custom moduleSpecifier when provided', async () => {
-    const project = new Project({ useInMemoryFileSystem: true })
-    const file = project.createSourceFile('/com/example/foo.defs.ts')
-    const indexer = new DummyIndexer(docs)
-    const resolver = new RefResolver(docs[0], file, indexer, {
-      moduleSpecifier: (nsid) => `https://lex.example.com/${nsid}.ts`,
-    })
-
-    await resolver.resolve('com.example.bar#main')
-
-    const imports = file.getImportDeclarations()
-    expect(imports).toHaveLength(1)
-    expect(imports[0].getModuleSpecifierValue()).toBe(
-      'https://lex.example.com/com.example.bar.ts',
-    )
-  })
-})

package/src/ref-resolver.ts

@@ -1,437 +0,0 @@
-import assert from 'node:assert'
-import { join } from 'node:path'
-import { SourceFile } from 'ts-morph'
-import { LexiconDocument, LexiconIndexer } from '@atproto/lex-document'
-import {
-  isGlobalIdentifier,
-  isJsKeyword,
-  isSafeLocalIdentifier,
-  isValidJsIdentifier,
-} from './ts-lang.js'
-import {
-  asRelativePath,
-  memoize,
-  startsWithLower,
-  toCamelCase,
-  toPascalCase,
-  ucFirst,
-} from './util.js'
-
-/**
- * Configuration options for the {@link RefResolver} class.
- */
-export type RefResolverOptions = {
-  /**
-   * The file extension to use for import specifiers when resolving
-   * external references.
-   *
-   * @default '.js'
-   */
-  importExt?: string
-  moduleSpecifier?: (nsid: string) => string
-}
-
-/**
- * Represents a resolved lexicon reference as TypeScript identifiers.
- *
- * Contains the variable name (for runtime schema) and type name (for
- * TypeScript type) that can be used to reference a lexicon definition.
- */
-export type ResolvedRef = {
-  /**
-   * The variable name for the runtime schema.
-   *
-   * For local definitions, this is a simple identifier.
-   * For external definitions, this may be a qualified name like `ns.varName`
-   * or bracket notation like `ns["varName"]` for unsafe identifiers.
-   */
-  varName: string
-  /**
-   * The type name for the TypeScript type alias.
-   *
-   * Always a valid TypeScript identifier, either simple or qualified.
-   */
-  typeName: string
-}
-
-/**
- * Resolves lexicon references to TypeScript identifiers.
- *
- * This class handles the resolution of `ref` types in lexicon documents,
- * converting lexicon reference strings (like `com.example.foo#bar`) into
- * valid TypeScript identifiers. It automatically manages:
- *
- * - Local references within the same document
- * - External references to other lexicon documents
- * - Import statement generation for external references
- * - Conflict avoidance with keywords, globals, and existing declarations
- *
- * Results are memoized to ensure consistent identifiers for the same
- * reference throughout a file.
- *
- * @example
- * ```ts
- * const resolver = new RefResolver(doc, sourceFile, indexer, options)
- *
- * // Resolve a local reference
- * const local = await resolver.resolve('#myDef')
- *
- * // Resolve an external reference
- * const external = await resolver.resolve('com.example.other#def')
- * ```
- */
-export class RefResolver {
-  constructor(
-    private doc: LexiconDocument,
-    private file: SourceFile,
-    private indexer: LexiconIndexer,
-    private options: RefResolverOptions,
-  ) {}
-
-  public readonly resolve = memoize(
-    async (ref: string): Promise<ResolvedRef> => {
-      const [nsid, hash = 'main'] = ref.split('#')
-
-      if (nsid === '' || nsid === this.doc.id) {
-        return this.resolveLocal(hash)
-      } else {
-        // @NOTE: Normalize (#main fragment) to ensure proper memoization
-        const fullRef = `${nsid}#${hash}`
-        return this.resolveExternal(fullRef)
-      }
-    },
-  )
-
-  #defCounters = new Map<string, number>()
-  private nextSafeDefinitionIdentifier(name: string) {
-    // use camelCase version of the hash as base name
-    const nameSafe =
-      startsWithLower(name) && isValidJsIdentifier(name)
-        ? name
-        : toCamelCase(name).replace(/^[0-9]+/g, '') || 'def'
-
-    const count = this.#defCounters.get(nameSafe) ?? 0
-    this.#defCounters.set(nameSafe, count + 1)
-
-    // @NOTE We don't need to check against local declarations in the file here
-    // since we are using a naming system that should guarantee no other
-    // identifier has a <nameSafe>$<number> format ("$" cannot appear in
-    // hashes so only *we* are generating such identifiers).
-
-    const identifier = `${nameSafe}$${count}`
-
-    assert(
-      isValidJsIdentifier(identifier),
-      `Unable to generate safe identifier for: "${name}"`,
-    )
-
-    return identifier
-  }
-
-  /**
-   * Resolves a local definition hash to TypeScript identifiers.
-   *
-   * This method generates safe, non-conflicting identifiers for definitions
-   * within the current document. It handles edge cases like:
-   * - Hash names that are JavaScript keywords
-   * - Hash names that conflict with global identifiers
-   * - Multiple hashes that would produce the same identifier
-   *
-   * @param hash - The definition hash (e.g., 'main', 'record', 'myType')
-   * @returns A promise resolving to the TypeScript identifiers
-   * @throws Error if the hash does not exist in the document
-   * @throws Error if conflicting type names are detected
-   *
-   * @note The returned `typeName` and `varName` are *both* guaranteed to be
-   * valid TypeScript identifiers.
-   */
-  public readonly resolveLocal = memoize(
-    async (hash: string): Promise<ResolvedRef> => {
-      const hashes = Object.keys(this.doc.defs)
-
-      if (!hashes.includes(hash)) {
-        throw new Error(`Definition ${hash} not found in ${this.doc.id}`)
-      }
-
-      // Because we are using predictable "public" identifiers for type names,
-      // we need to ensure there are no conflicts between different definitions
-      // in the same lexicon document.
-      //
-      // @NOTE It should be possible to implement a way to generate
-      // non-conflicting type names for all public (type) identifiers in a
-      // project. However, this would add a lot of complexity to the code
-      // generation process, and the likelihood of such conflicts happening in
-      // practice is very low, so we opt for a simpler approach of just throwing
-      // an error if a conflict is detected.
-      const pub = getPublicIdentifiers(hash)
-      for (const otherHash of hashes) {
-        if (otherHash === hash) continue
-        const otherPub = getPublicIdentifiers(otherHash)
-        if (otherPub.typeName === pub.typeName) {
-          throw new Error(
-            `Conflicting type names for definitions #${hash} and #${otherHash} in ${this.doc.id}`,
-          )
-        }
-      }
-
-      // Try to keep and identifier that resembles the original hash as identifier
-      const safeIdentifier = asSafeDefinitionIdentifier(hash)
-
-      // If the safe identifier is not conflicting with other definition names,
-      // or reserved words, we can use it as-is. Otherwise, we need to generate
-      // a unique safe identifier.
-      const varName = safeIdentifier
-        ? !hashes.some((otherHash) => {
-            if (otherHash === hash) return false
-            const otherIdentifier = asSafeDefinitionIdentifier(otherHash)
-            return otherIdentifier === safeIdentifier
-          })
-          ? // Safe identifier can be used as-is as it does not conflict with
-            // other definition names
-            safeIdentifier
-          : // In order to keep identifiers stable, we use the safe identifier
-            // as base, and append a counter to avoid conflicts
-            this.nextSafeDefinitionIdentifier(safeIdentifier)
-        : // hash only contained unsafe characters, generate a safe one
-          this.nextSafeDefinitionIdentifier(hash)
-
-      const typeName = ucFirst(varName)
-      assert(isSafeLocalIdentifier(typeName), 'Expected safe type identifier')
-      assert(varName !== typeName, 'Variable and type name should be different')
-
-      return { varName, typeName }
-    },
-  )
-
-  /**
-   * @note Since this is a memoized function, and is used to generate the name
-   * of local variables, we should avoid returning different results for
-   * similar, but non strictly equal, inputs (eg. normalized / non-normalized).
-   * @see {@link resolve}
-   */
-  private readonly resolveExternal = memoize(
-    async (fullRef: string): Promise<ResolvedRef> => {
-      const [nsid, hash] = fullRef.split('#')
-      const moduleSpecifier = this.options.moduleSpecifier
-        ? this.options.moduleSpecifier(nsid)
-        : `${asRelativePath(
-            this.file.getDirectoryPath(),
-            join('/', ...nsid.split('.')),
-          )}.defs${this.options.importExt ?? '.js'}`
-
-      // Lets first make sure the referenced lexicon exists
-      const srcDoc = await this.indexer.get(nsid)
-      const srcDef = Object.hasOwn(srcDoc.defs, hash) ? srcDoc.defs[hash] : null
-      if (!srcDef) {
-        throw new Error(
-          `Missing def "${hash}" in "${nsid}" (referenced from ${this.doc.id})`,
-        )
-      }
-
-      const publicIds = getPublicIdentifiers(hash)
-
-      if (!isValidJsIdentifier(publicIds.typeName)) {
-        // If <typeName> is not a valid identifier, we cannot access the type
-        // using dot notation (<nsIdentifier>.<typeName>). Note that, unlike js
-        // variables, types cannot be accessed using string indexing (like:
-        // <nsIdentifier>['<typeName>']) because it generates TypeScript errors:
-        //
-        // > "Cannot use namespace '<nsIdentifier>' as a type."
-
-        // Instead the generated code should look like:
-        // import { "<unsafeTypeName>" as <safeIdentifier> } from './<moduleSpecifier>.js'
-
-        // Because it requires more complex management of local variables names,
-        // and we don't expect this to actually happen with properly designed
-        // lexicons documents, we do not support this for now.
-
-        throw new Error(
-          'Import of definitions with unsafe type names is not supported',
-        )
-      }
-
-      // import * as <nsIdentifier> from './<moduleSpecifier>.js'
-      const nsIdentifier = this.getNsIdentifier(nsid, moduleSpecifier)
-
-      return {
-        varName: isValidJsIdentifier(publicIds.varName)
-          ? `${nsIdentifier}.${publicIds.varName}`
-          : `${nsIdentifier}[${JSON.stringify(publicIds.varName)}]`,
-        typeName: `${nsIdentifier}.${publicIds.typeName}`,
-      }
-    },
-  )
-
-  private getNsIdentifier(nsid: string, moduleSpecifier: string) {
-    const namespaceImportDeclaration =
-      this.file.getImportDeclaration(
-        (imp) =>
-          !imp.isTypeOnly() &&
-          imp.getModuleSpecifierValue() === moduleSpecifier &&
-          imp.getNamespaceImport() != null,
-      ) ||
-      this.file.addImportDeclaration({
-        moduleSpecifier,
-        namespaceImport: this.computeSafeNamespaceIdentifierFor(nsid),
-      })
-
-    return namespaceImportDeclaration.getNamespaceImport()!.getText()
-  }
-
-  #nsIdentifiersCounters = new Map<string, number>()
-  private computeSafeNamespaceIdentifierFor(nsid: string) {
-    const baseName = nsidToIdentifier(nsid) || 'NS'
-
-    let name = baseName
-    while (this.isConflictingIdentifier(name)) {
-      const count = this.#nsIdentifiersCounters.get(baseName) ?? 0
-      this.#nsIdentifiersCounters.set(baseName, count + 1)
-      name = `${baseName}$$${count}`
-    }
-
-    return name
-  }
-
-  private isConflictingIdentifier(name: string) {
-    return (
-      this.conflictsWithKeywords(name) ||
-      this.conflictsWithUtils(name) ||
-      this.conflictsWithLocalDefs(name) ||
-      this.conflictsWithLocalDeclarations(name) ||
-      this.conflictsWithImports(name)
-    )
-  }
-
-  private conflictsWithKeywords(name: string) {
-    return isJsKeyword(name) || isGlobalIdentifier(name)
-  }
-
-  private conflictsWithUtils(name: string) {
-    // Do not allow "Main" as imported ns identifier since it has a special
-    // meaning in the context of lexicon definitions.
-    if (name === 'Main') return true
-
-    // When "useRecordExport" returns true, an export named "Record" will be
-    // used in addition to the hash named export. So we need to make sure both
-    // names are not conflicting with local variables.
-    if (name === 'Record') return true
-
-    // Utility functions generated for lexicon schemas are prefixed with "$"
-    return name.startsWith('$')
-  }
-
-  private conflictsWithLocalDefs(name: string) {
-    return Object.keys(this.doc.defs).some((hash) => {
-      const identifier = toCamelCase(hash)
-
-      // A safe identifier will be generated, no risk of conflict.
-      if (!identifier) return false
-
-      // The imported name conflicts with a local definition name
-      if (identifier === name || `_${identifier}` === name) return true
-
-      // The imported name conflicts with the type name of a local definition
-      const typeName = ucFirst(identifier)
-      if (typeName === name || `_${typeName}` === name) return true
-
-      return false
-    })
-  }
-
-  private conflictsWithLocalDeclarations(name: string) {
-    return (
-      this.file.getVariableDeclarations().some((v) => v.getName() === name) ||
-      this.file
-        .getVariableStatements()
-        .some((vs) => vs.getDeclarations().some((d) => d.getName() === name)) ||
-      this.file.getTypeAliases().some((t) => t.getName() === name) ||
-      this.file.getInterfaces().some((i) => i.getName() === name) ||
-      this.file.getClasses().some((c) => c.getName() === name) ||
-      this.file.getFunctions().some((f) => f.getName() === name) ||
-      this.file.getEnums().some((e) => e.getName() === name)
-    )
-  }
-
-  private conflictsWithImports(name: string) {
-    return this.file.getImportDeclarations().some(
-      (imp) =>
-        // import name from '...'
-        imp.getDefaultImport()?.getText() === name ||
-        // import * as name from '...'
-        imp.getNamespaceImport()?.getText() === name ||
-        imp.getNamedImports().some(
-          (named) =>
-            // import { name } from '...'
-            // import { foo as name } from '...'
-            (named.getAliasNode()?.getText() ?? named.getName()) === name,
-        ),
-    )
-  }
-}
-
-/**
- * @see {@link https://atproto.com/specs/nsid NSID syntax spec}
- */
-function nsidToIdentifier(nsid: string) {
-  const parts = nsid.split('.')
-
-  // By default, try to keep only to the last two segments of the NSID as
-  // contextual information. If those do not form a safe identifier (typically
-  // because they start with a digit), try with more segments until we reach the
-  // full NSID.
-  for (let i = 2; i < parts.length; i++) {
-    const identifier = toPascalCase(parts.slice(-i).join('.'))
-    if (isSafeLocalIdentifier(identifier)) return identifier
-  }
-
-  return undefined
-}
-
-/**
- * Generates predictable public identifiers for a given definition hash.
- *
- * This function creates the "public" names that will be exported from
- * generated files. The variable name uses the original hash, while the
- * type name is converted to PascalCase.
- *
- * @param hash - The definition hash (e.g., 'main', 'myType')
- * @returns The public identifiers for the definition
- *
- * @note The returned `typeName` is guaranteed to be a valid TypeScript
- * identifier. `varName` may not be a valid identifier (eg. if the hash contains
- * unsafe characters), and may need to be accessed using string indexing.
- */
-export function getPublicIdentifiers(hash: string): ResolvedRef {
-  const varName = hash
-
-  // @NOTE we try to circumvent the issue of unsafe type names described in
-  // `RefResolver.resolveExternal` by ensuring that type names are always safe
-  // identifiers, even if it means changing them from the original hash.
-  let typeName = toPascalCase(hash)
-
-  if (varName === typeName || !isValidJsIdentifier(typeName)) {
-    typeName = `TypeOf${typeName}`
-  }
-
-  assert(
-    isValidJsIdentifier(typeName),
-    `Unable to generate a predictable safe identifier for "${hash}"`,
-  )
-
-  return { varName, typeName }
-}
-
-function asSafeDefinitionIdentifier(name: string) {
-  if (
-    startsWithLower(name) &&
-    isSafeLocalIdentifier(name) &&
-    isSafeLocalIdentifier(ucFirst(name))
-  ) {
-    return name
-  }
-  const camel = toCamelCase(name)
-  if (isSafeLocalIdentifier(camel) && isSafeLocalIdentifier(ucFirst(camel))) {
-    return camel
-  }
-  return undefined
-}